From 42ff60f6f0979447eb1b1a76224e87e5738efd31 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 24 Apr 2026 19:25:36 +0100 Subject: [PATCH 1/9] Align data preprocessing with configured time unit --- macro_data/data_wrapper.py | 54 ++++++++++- .../default_synthetic_central_government.py | 18 +++- macro_data/processing/synthetic_country.py | 20 +++- .../hfcs_household_tools.py | 56 ++++++++++- .../hfcs_individual_tools.py | 14 ++- .../hfcs_synthetic_population.py | 5 + macro_data/readers/default_readers.py | 51 ++++++++-- .../economic_data/world_bank_reader.py | 4 +- macro_data/readers/exogenous_data.py | 93 ++++++++++++++++++- .../readers/population_data/hfcs_reader.py | 26 ++++++ macromodel/country/country.py | 1 + macromodel/exogenous/exogenous.py | 49 ++++++++-- macromodel/exogenous/exogenous_ts.py | 18 +++- .../unit/test_readers/test_exogenous.py | 52 ++++++++++- .../unit/test_exogenous/test_exogenous.py | 39 ++++++++ 15 files changed, 453 insertions(+), 47 deletions(-) diff --git a/macro_data/data_wrapper.py b/macro_data/data_wrapper.py index ef2d9126..52990988 100644 --- a/macro_data/data_wrapper.py +++ b/macro_data/data_wrapper.py @@ -62,10 +62,44 @@ EmissionsData, EmissionsEnergyFactors, ) -from macro_data.readers.exogenous_data import ExogenousCountryData +from macro_data.readers.exogenous_data import ExogenousCountryData, convert_growth_rates_to_model_period from macro_data.readers.io_tables.icio_reader import ICIOReader +def _quarter_start_date(year: int, quarter: int) -> pd.Timestamp: + return pd.Timestamp(year, 3 * (quarter - 1) + 1, 1) + + +def _period_start_for_date(index: pd.Index, date: pd.Timestamp) -> pd.Timestamp: + index = pd.DatetimeIndex(index).sort_values() + if date in index: + return date + + prior_dates = index[index <= date] + if len(prior_dates) > 0: + return prior_dates[-1] + + later_dates = index[index > date] + if len(later_dates) > 0: + return later_dates[0] + + raise ValueError("Cannot select a period from an empty calibration index.") + + +def _period_values_at_date(data: pd.DataFrame, date: pd.Timestamp) -> pd.Series: + if date in data.index: + return data.loc[date] + + return ( + data.reindex(data.index.union(pd.DatetimeIndex([date]))) + .sort_index() + .interpolate(method="time") + .ffill() + .bfill() + .loc[date] + ) + + @dataclass class DataWrapper: """ @@ -181,6 +215,7 @@ def from_config( year = configuration.year quarter = configuration.quarter + yearly_factor = 12 / configuration.time_unit scale_dict = {country: configuration.country_configs[country].scale for country in country_names} @@ -211,6 +246,9 @@ def from_config( use_disagg_can_2014_reader=configuration.can_disaggregation, use_provincial_can_reader=use_provincial_can_reader, regions_dict=regions_dict, + allow_missing_emissions=allow_missing_emissions, + yearly_factor=yearly_factor, + simulation_quarter=quarter, ) if regions_dict: @@ -241,6 +279,7 @@ def from_config( readers=readers, country_names=country_names, single_firm_per_industry=single_firm_dict, + yearly_factor=yearly_factor, ) year_range = 1 if single_hfcs_survey else 10 @@ -255,6 +294,7 @@ def from_config( quarter=quarter, industry_vectors=industry_data[country]["industry_vectors"], proxy_country=proxy_country_dict.get(country, None), + time_unit=configuration.time_unit, ) for country in country_names } @@ -269,6 +309,7 @@ def from_config( inflation = readers.imf_reader.get_inflation(proxy_country) if inflation is None: inflation = readers.world_bank.get_inflation(proxy_country) + inflation = convert_growth_rates_to_model_period(inflation, configuration.time_unit) proxy_inflation[country] = inflation else: proxy_inflation[country] = None @@ -423,7 +464,8 @@ def calibration_before(self): year = self.configuration.year quarter = self.configuration.quarter calibration_index = self.calibration_data.index - calibration_before_index = calibration_index[calibration_index < f"{year}-Q{quarter}"] + start_date = _period_start_for_date(calibration_index, _quarter_start_date(year, quarter)) + calibration_before_index = calibration_index[calibration_index < start_date] return self.calibration_data.loc[calibration_before_index] @property @@ -438,7 +480,8 @@ def calibration_during(self): year = self.configuration.year quarter = self.configuration.quarter calibration_index = self.calibration_data.index - calibration_during_index = calibration_index[calibration_index == f"{year}-Q{quarter}"] + start_date = _period_start_for_date(calibration_index, _quarter_start_date(year, quarter)) + calibration_during_index = calibration_index[calibration_index == start_date] return self.calibration_data.loc[calibration_during_index] @@ -472,8 +515,9 @@ def add_row_to_calibration( all_exports = calibration_data.xs("Exports (Value)", axis=1, level=1) all_imports = calibration_data.xs("Imports (Value)", axis=1, level=1) - scaled_exports = all_exports / all_exports.loc[f"{year}-Q{quarter}"].iloc[0] - scaled_imports = all_imports / all_imports.loc[f"{year}-Q{quarter}"].iloc[0] + base_date = _quarter_start_date(year, quarter) + scaled_exports = all_exports / _period_values_at_date(all_exports, base_date) + scaled_imports = all_imports / _period_values_at_date(all_imports, base_date) total_country_exports = sum( [country_scaled_exports(country, industry_data, scaled_exports) for country in countries] diff --git a/macro_data/processing/synthetic_central_government/default_synthetic_central_government.py b/macro_data/processing/synthetic_central_government/default_synthetic_central_government.py index 0c081ecd..b9078792 100644 --- a/macro_data/processing/synthetic_central_government/default_synthetic_central_government.py +++ b/macro_data/processing/synthetic_central_government/default_synthetic_central_government.py @@ -100,6 +100,7 @@ def from_readers( year_range: int = 10, regression_window: int = 48, equity_injection: float = 0.0, + yearly_factor: float = 4.0, ) -> SyntheticCentralGovernment: """Create a preprocessed central government data container using standard data sources. @@ -129,7 +130,11 @@ def from_readers( if country_exogenous_data is not None: # if exogenous data is available, use it to fit the benefits models benefits_inflation_data = readers.get_benefits_inflation_data( - country_name, year_min=year - year_range, year_max=year, exogenous_data=country_exogenous_data + country_name, + year_min=year - year_range, + year_max=year, + exogenous_data=country_exogenous_data, + yearly_factor=yearly_factor, ) unemployment_benefits_model = build_unemployment_model( benefits_inflation_data, regression_window=regression_window @@ -160,8 +165,15 @@ def from_readers( # if exogenous data is not available, set the benefits models to None unemployment_benefits_model = None other_benefits_model = None - current_unemployment_benefits = readers.get_total_unemployment_benefits_lcu(country_name, year) - current_other_benefits = readers.get_total_benefits_lcu(country_name, year) - current_unemployment_benefits + current_unemployment_benefits = readers.get_total_unemployment_benefits_lcu( + country_name, + year, + yearly_factor=yearly_factor, + ) + current_other_benefits = ( + readers.get_total_benefits_lcu(country_name, year, yearly_factor=yearly_factor) + - current_unemployment_benefits + ) # TODO: debt in USD or in local currency? diff --git a/macro_data/processing/synthetic_country.py b/macro_data/processing/synthetic_country.py index 210d5e98..1fa7524e 100644 --- a/macro_data/processing/synthetic_country.py +++ b/macro_data/processing/synthetic_country.py @@ -215,7 +215,14 @@ def eu_synthetic_country( This method should only be used for EU member countries. For non-EU countries, use proxied_synthetic_country instead. """ - central_government = DefaultSyntheticCGovernment.from_readers(readers, country, year, year_range=year_range) + yearly_factor = 12 / time_unit + central_government = DefaultSyntheticCGovernment.from_readers( + readers, + country, + year, + year_range=year_range, + yearly_factor=yearly_factor, + ) total_unemployment_benefits = central_government.central_gov_data["Total Unemployment Benefits"].values[0] @@ -245,6 +252,7 @@ def eu_synthetic_country( total_unemployment_benefits=total_unemployment_benefits, country_name_short=country.to_two_letter_code(), exogenous_data=exogenous_country_data, + yearly_factor=yearly_factor, ) firms = DefaultSyntheticFirms.from_readers( @@ -394,7 +402,14 @@ def proxied_synthetic_country( Returns: SyntheticCountry: Initialized synthetic country instance """ - central_government = DefaultSyntheticCGovernment.from_readers(readers, country, year, year_range=year_range) + yearly_factor = 12 / time_unit + central_government = DefaultSyntheticCGovernment.from_readers( + readers, + country, + year, + year_range=year_range, + yearly_factor=yearly_factor, + ) total_unemployment_benefits = central_government.central_gov_data["Total Unemployment Benefits"].values[0] @@ -433,6 +448,7 @@ def proxied_synthetic_country( proxied_country=country, quarter=quarter, exogenous_data=exogenous_country_data, + yearly_factor=yearly_factor, ) firms = DefaultSyntheticFirms.from_readers( diff --git a/macro_data/processing/synthetic_population/hfcs_household_tools.py b/macro_data/processing/synthetic_population/hfcs_household_tools.py index 27bbcd9a..cc82865b 100644 --- a/macro_data/processing/synthetic_population/hfcs_household_tools.py +++ b/macro_data/processing/synthetic_population/hfcs_household_tools.py @@ -3,6 +3,23 @@ from macro_data.util.imputation import apply_iterative_imputer +MONTHLY_HFCS_CASH_FLOW_COLUMNS = [ + "Rent Paid", + "Rent Paid for Partially Owned Dwelling", + "Mortgage Payment on Main Residence 1", + "Mortgage Payment on Main Residence 2", + "Mortgage Payment on Main Residence 3", + "Additional Mortgage Payments on Main Residence", + "Other Property Loan Payments", + "Consumer Loan Payment 1", + "Consumer Loan Payment 2", + "Consumer Loan Payment 3", + "Additional Consumer Loan Payments", + "Pension Contributions", + "Private Transfers Given", + "Health Insurance Payments", +] + def get_household_type(ages: np.ndarray) -> int: """ @@ -88,6 +105,7 @@ def set_household_housing_data( scale: int, rent_as_fraction_of_unemployment_rate: float, unemployment_benefits_by_capita: float, + yearly_factor: float = 4.0, ) -> pd.DataFrame: """ Sets the housing data for each household in the given DataFrame. @@ -115,7 +133,15 @@ def set_household_housing_data( # Rent paid and value of the household main residence social_housing_rent = rent_as_fraction_of_unemployment_rate * unemployment_benefits_by_capita - household_data = fill_rent(household_data, households_owning, households_renting, scale, social_housing_rent) + household_data = fill_rent( + household_data, + households_owning, + households_renting, + scale, + social_housing_rent, + yearly_factor=yearly_factor, + ) + household_data = rescale_monthly_hfcs_cash_flows(household_data, scale, yearly_factor) # Number of additional properties # household_data["Number of Properties other than Household Main Residence"].fillna(0, inplace=True) @@ -131,7 +157,13 @@ def set_household_housing_data( household_data = fix_property_values(household_data, household_without_additional_properties) # Rent received - household_data = fix_rent(household_data, household_without_additional_properties, scale, social_housing_rent) + household_data = fix_rent( + household_data, + household_without_additional_properties, + scale, + social_housing_rent, + yearly_factor=yearly_factor, + ) return household_data @@ -140,6 +172,7 @@ def fix_rent( household_without_additional_properties: pd.Series, scale: int, social_housing_rent: float, + yearly_factor: float = 4.0, ) -> pd.DataFrame: """ Adjusts the rental income of households based on specified parameters. @@ -154,7 +187,7 @@ def fix_rent( pd.DataFrame: The updated dataframe with adjusted rental income. """ household_data.loc[:, "Rental Income from Real Estate"] *= scale - household_data.loc[:, "Rental Income from Real Estate"] /= 12.0 + household_data.loc[:, "Rental Income from Real Estate"] /= yearly_factor household_data.loc[ household_data["Rental Income from Real Estate"] < social_housing_rent, "Rental Income from Real Estate", @@ -202,6 +235,7 @@ def fill_rent( households_renting: pd.Series, scale: int, social_housing_rent: float, + yearly_factor: float = 4.0, ) -> pd.DataFrame: """ Fill in missing values for rent paid and value of the main residence in the household data. @@ -217,7 +251,12 @@ def fill_rent( Returns: pd.DataFrame: Updated household data with filled-in values for rent paid and value of the main residence. """ - household_data.loc[:, "Rent Paid"] *= scale + monthly_factor = 12.0 / yearly_factor + if "Rent Paid for Partially Owned Dwelling" in household_data.columns: + household_data["Rent Paid"] = household_data["Rent Paid"].fillna(0.0) + household_data[ + "Rent Paid for Partially Owned Dwelling" + ].fillna(0.0) + household_data.loc[:, "Rent Paid"] *= scale * monthly_factor household_data.loc[:, "Value of the Main Residence"] *= scale household_data.loc[households_renting & (household_data["Rent Paid"] == 0.0), "Rent Paid"] = np.nan household_data.loc[ @@ -237,3 +276,12 @@ def fill_rent( "Rent Paid", ] = 0.0 return household_data + + +def rescale_monthly_hfcs_cash_flows(household_data: pd.DataFrame, scale: int, yearly_factor: float) -> pd.DataFrame: + """Convert imported monthly HFCS cash flows to model-period values.""" + monthly_factor = 12.0 / yearly_factor + columns = [col for col in MONTHLY_HFCS_CASH_FLOW_COLUMNS if col in household_data.columns and col != "Rent Paid"] + if columns: + household_data.loc[:, columns] *= scale * monthly_factor + return household_data diff --git a/macro_data/processing/synthetic_population/hfcs_individual_tools.py b/macro_data/processing/synthetic_population/hfcs_individual_tools.py index 409a3172..a5be6d65 100644 --- a/macro_data/processing/synthetic_population/hfcs_individual_tools.py +++ b/macro_data/processing/synthetic_population/hfcs_individual_tools.py @@ -15,6 +15,7 @@ def process_individual_data( unemployment_rate: float, participation_rate: float, n_firms_by_industry: list[int] | np.ndarray, + yearly_factor: float = 4.0, ) -> pd.DataFrame: """ Process individual data by performing various data cleaning and transformation steps. @@ -62,7 +63,10 @@ def process_individual_data( logging.warning("Total unemployment benefits not found, setting to 0.0") individual_data = fill_individual_employee_income( - individual_data, unemployment_benefits_by_individual=total_unemployment_benefits / n_unemployed, scale=scale + individual_data, + unemployment_benefits_by_individual=total_unemployment_benefits / n_unemployed, + scale=scale, + yearly_factor=yearly_factor, ) individual_data = set_individual_unemployed_income( individual_data, unemployment_benefits_by_individual=total_unemployment_benefits / n_unemployed @@ -551,7 +555,10 @@ def select_employed_in_industry(individual_data: pd.DataFrame, industry: int) -> def fill_individual_employee_income( - individual_data: pd.DataFrame, unemployment_benefits_by_individual: float, scale: int + individual_data: pd.DataFrame, + unemployment_benefits_by_individual: float, + scale: int, + yearly_factor: float = 4.0, ) -> pd.DataFrame: """ Fills the 'Employee Income' column in the individual_data DataFrame for employed individuals. @@ -586,8 +593,7 @@ def fill_individual_employee_income( # Rescale that individual_data.loc[:, "Employee Income"] *= scale - # Monthly! - individual_data.loc[:, "Employee Income"] /= 4.0 + individual_data.loc[:, "Employee Income"] /= yearly_factor # Employee income is at least the unemployment rate is_employed = individual_data["Activity Status"] == 1 diff --git a/macro_data/processing/synthetic_population/hfcs_synthetic_population.py b/macro_data/processing/synthetic_population/hfcs_synthetic_population.py index 8cf5059f..a919e344 100644 --- a/macro_data/processing/synthetic_population/hfcs_synthetic_population.py +++ b/macro_data/processing/synthetic_population/hfcs_synthetic_population.py @@ -144,6 +144,7 @@ def __init__( consumption_weights: np.ndarray, consumption_weights_by_income: np.ndarray, investment: np.ndarray, + yearly_factor: float = 4.0, ): saving_rates_model = LinearRegression() social_transfers_model = LinearRegression() @@ -164,6 +165,7 @@ def __init__( saving_rates_model, social_transfers_model, wealth_distribution_model, + yearly_factor=yearly_factor, ) # TODO rent as fraction of unemployment rate seems to be a parameter of government functions @@ -241,6 +243,7 @@ def from_readers( unemployment_rate, participation_rate, n_firms_by_industry, + yearly_factor=yearly_factor, ) n_unemployed = np.sum(individual_data["Activity Status"] == 2) @@ -276,6 +279,7 @@ def from_readers( scale, rent_as_fraction_of_unemployment_rate, unemployment_benefits_by_capita=total_unemployment_benefits / n_unemployed, + yearly_factor=yearly_factor, ) # initialise fields to nans, will be filled later when computing wealth @@ -319,6 +323,7 @@ def from_readers( consumption_weights_by_income=consumption_weights_by_income, coefficient_fa_income=0.0, investment=investment, + yearly_factor=yearly_factor, ) def restrict(self) -> None: diff --git a/macro_data/readers/default_readers.py b/macro_data/readers/default_readers.py index ad36e38f..277255aa 100644 --- a/macro_data/readers/default_readers.py +++ b/macro_data/readers/default_readers.py @@ -150,6 +150,17 @@ def default_paths(cls, raw_data_path: Path, icio_years: Iterable[int]): # return paths +def _time_unit_from_yearly_factor(yearly_factor: float) -> int: + if yearly_factor <= 0: + raise ValueError("`yearly_factor` must be positive.") + + time_unit = 12 / yearly_factor + rounded_time_unit = round(time_unit) + if not np.isclose(time_unit, rounded_time_unit): + raise ValueError("`yearly_factor` must imply an integer simulation time unit.") + return int(rounded_time_unit) + + @dataclass class DataReaders: """Centralized management of all data readers for the model. @@ -212,6 +223,9 @@ def from_raw_data( use_disagg_can_2014_reader: bool = False, use_provincial_can_reader: bool = False, regions_dict: dict[Country, list[Region]] = None, + allow_missing_emissions: bool = False, + yearly_factor: float = 4.0, + simulation_quarter: int = 1, ): if regions_dict: all_regions = [region for regions in regions_dict.values() for region in regions] @@ -255,6 +269,7 @@ def get_investment_year(year: int, country_names_: Optional[list[Country | Regio investment_fractions=get_investment_year(year), proxy_country_dict=proxy_country_dict, aggregation_type="Aggregate" if aggregate_industries else "All", + yearly_factor=yearly_factor, ) for year in all_years } @@ -396,6 +411,7 @@ def get_investment_year(year: int, country_names_: Optional[list[Country | Regio sea_reader=wiod_sea, country_names=country_names, regions_dict=regions_dict, + yearly_factor=yearly_factor, ) add_investment_matrix_to_icio( @@ -403,6 +419,7 @@ def get_investment_year(year: int, country_names_: Optional[list[Country | Regio sea_reader=wiod_sea, country_names=country_names, regions_dict=regions_dict, + yearly_factor=yearly_factor, ) match_iot_with_sea( @@ -410,6 +427,7 @@ def get_investment_year(year: int, country_names_: Optional[list[Country | Regio sea_reader=wiod_sea, country_names=country_names, regions_dict=regions_dict, + yearly_factor=yearly_factor, ) oecd_econ = OECDEconData( @@ -430,17 +448,23 @@ def get_investment_year(year: int, country_names_: Optional[list[Country | Regio ecb_reader = ECBReader(path=datapaths.ecb_path) all_countries = list(set(country_names).union(set(proxy_country_dict.values()))) + time_unit = _time_unit_from_yearly_factor(yearly_factor) compustat_firms = CompustatFirmsReader.from_raw_data( year=simulation_year, - quarter=1, + quarter=simulation_quarter, countries=all_countries, raw_annual_path=datapaths.compustat_firms_annual_path, raw_quarterly_path=datapaths.compustat_firms_quarterly_path, + time_unit=time_unit, ) compustat_banks = CompustatBanksReader.from_raw_data( - year=simulation_year, quarter=1, raw_quarterly_path=datapaths.compustat_banks_path, countries=all_countries + year=simulation_year, + quarter=simulation_quarter, + raw_quarterly_path=datapaths.compustat_banks_path, + countries=all_countries, + time_unit=time_unit, ) if prune_date: @@ -535,7 +559,12 @@ def get_exogenous_data(self, country_name: Country) -> Optional[dict[str, Any]]: return None def get_benefits_inflation_data( - self, country_name: Country, year_min: int, year_max: int, exogenous_data: dict[str, Any] + self, + country_name: Country, + year_min: int, + year_max: int, + exogenous_data: dict[str, Any], + yearly_factor: float = 4.0, ) -> pd.DataFrame: """Calculate benefits and inflation data for a country. @@ -555,12 +584,12 @@ def get_benefits_inflation_data( years = range(year_min, year_max) unemp = [ self.oecd_econ.unemployment_benefits_gdp_pct(country_name, year) - * self.world_bank.get_current_scaled_gdp(country_name, year) + * self.world_bank.get_current_scaled_gdp(country_name, year, rescale_factor=yearly_factor) for year in years ] other = [ self.oecd_econ.all_benefits_gdp_pct(country_name, year) - * self.world_bank.get_current_scaled_gdp(country_name, year) + * self.world_bank.get_current_scaled_gdp(country_name, year, rescale_factor=yearly_factor) - unemp[i] for i, year in enumerate(years) ] @@ -586,7 +615,7 @@ def get_benefits_inflation_data( data = pd.merge_asof(data, unemployment_rate, left_index=True, right_index=True) return data - def get_total_benefits_lcu(self, country_name: Country, year: int) -> float: + def get_total_benefits_lcu(self, country_name: Country, year: int, yearly_factor: float = 4.0) -> float: """Calculate total benefits in local currency units. This method computes the total benefits (including unemployment and other benefits) @@ -599,11 +628,13 @@ def get_total_benefits_lcu(self, country_name: Country, year: int) -> float: Returns: float: Total benefits in local currency units """ - return self.oecd_econ.all_benefits_gdp_pct(country_name, year) * self.world_bank.get_current_scaled_gdp( + return self.oecd_econ.all_benefits_gdp_pct( country_name, year - ) + ) * self.world_bank.get_current_scaled_gdp(country_name, year, rescale_factor=yearly_factor) - def get_total_unemployment_benefits_lcu(self, country_name: Country, year: int) -> float: + def get_total_unemployment_benefits_lcu( + self, country_name: Country, year: int, yearly_factor: float = 4.0 + ) -> float: """Calculate total unemployment benefits in local currency units. This method computes the total unemployment benefits for a country @@ -618,7 +649,7 @@ def get_total_unemployment_benefits_lcu(self, country_name: Country, year: int) """ return self.oecd_econ.unemployment_benefits_gdp_pct( country_name, year - ) * self.world_bank.get_current_scaled_gdp(country_name, year) + ) * self.world_bank.get_current_scaled_gdp(country_name, year, rescale_factor=yearly_factor) def get_govt_debt_lcu(self, country: Country, year: int) -> float: """Calculate government debt in local currency units. diff --git a/macro_data/readers/economic_data/world_bank_reader.py b/macro_data/readers/economic_data/world_bank_reader.py index 25f282e1..ee2535ad 100644 --- a/macro_data/readers/economic_data/world_bank_reader.py +++ b/macro_data/readers/economic_data/world_bank_reader.py @@ -476,8 +476,8 @@ def get_inflation(self, country: str) -> pd.DataFrame: "PPI Inflation": vals_ppi, }, ) - data_df["CPI Inflation"] = np.log(data_df["CPI Inflation"] / data_df["CPI Inflation"].shift(1)) - data_df["PPI Inflation"] = np.log(data_df["PPI Inflation"] / data_df["PPI Inflation"].shift(1)) + data_df["CPI Inflation"] = data_df["CPI Inflation"] / data_df["CPI Inflation"].shift(1) - 1.0 + data_df["PPI Inflation"] = data_df["PPI Inflation"] / data_df["PPI Inflation"].shift(1) - 1.0 data_df.index = [pd.Timestamp(int(ind[0:4]), 3 * int(ind[6]) - 2, 1) for ind in data_df.index] # noqa return data_df.astype(float) diff --git a/macro_data/readers/exogenous_data.py b/macro_data/readers/exogenous_data.py index eb77624f..916833f8 100644 --- a/macro_data/readers/exogenous_data.py +++ b/macro_data/readers/exogenous_data.py @@ -26,12 +26,16 @@ def from_data_readers( year: int, quarter: int, proxy_country: Optional[Country] = None, + time_unit: int = 3, ): + validate_time_unit(time_unit) inflation = readers.imf_reader.get_inflation(country_name) if inflation is None: inflation = readers.world_bank.get_inflation(country_name) + inflation = convert_growth_rates_to_model_period(inflation, time_unit) national_accounts_growth = readers.get_national_accounts_growth(country_name) + national_accounts_growth = convert_growth_rates_to_model_period(national_accounts_growth, time_unit) if proxy_country is None: capital_formation_tax = readers.eurostat.taxrate_on_capital_formation(country_name, year) @@ -49,9 +53,10 @@ def from_data_readers( quarter, ) - labour_stats = prepare_labour_stats(country_name, readers) + labour_stats = prepare_labour_stats(country_name, readers, time_unit=time_unit) house_price_index = readers.oecd_econ.get_house_price_index(country_name) + house_price_index = convert_growth_rates_to_model_period(house_price_index, time_unit) return cls( country_name=country_name, @@ -88,7 +93,73 @@ def get_calibration_data(self, year: int, quarter: int): return country_data -def prepare_labour_stats(country_name: Country, readers: DataReaders): +def validate_time_unit(time_unit: int) -> None: + if time_unit <= 0 or 12 % time_unit != 0: + raise ValueError("time_unit must be a positive divisor of 12.") + + +def convert_growth_rates_to_model_period(data: pd.DataFrame | pd.Series, time_unit: int): + """Convert quarterly growth/rate series to the model period.""" + validate_time_unit(time_unit) + if time_unit == 3: + return data + + is_series = isinstance(data, pd.Series) + df = data.to_frame() if is_series else data.copy() + df.index = pd.to_datetime(df.index) + df = df.sort_index().astype(float) + + monthly_rows = [] + monthly_index = [] + for period_start, row in df.iterrows(): + monthly_row = (1.0 + row) ** (1.0 / 3.0) - 1.0 + for month_offset in range(3): + monthly_rows.append(monthly_row) + monthly_index.append(period_start + pd.DateOffset(months=month_offset)) + + monthly = pd.DataFrame(monthly_rows, index=pd.DatetimeIndex(monthly_index), columns=df.columns) + groups = np.arange(len(monthly)) // time_unit + period_index = monthly.index.to_series().groupby(groups).first() + + def compound_period(values: pd.Series) -> float: + if values.isna().any(): + return np.nan + return float((1.0 + values).prod() - 1.0) + + converted = monthly.groupby(groups).agg(compound_period) + converted.index = pd.DatetimeIndex(period_index.values) + return converted.iloc[:, 0] if is_series else converted + + +def convert_levels_to_model_period(data: pd.DataFrame | pd.Series, time_unit: int): + """Interpolate quarterly level/ratio series to model-period starts.""" + validate_time_unit(time_unit) + if time_unit == 3: + return data + + is_series = isinstance(data, pd.Series) + df = data.to_frame() if is_series else data.copy() + df.index = pd.to_datetime(df.index) + df = df.sort_index().astype(float) + + monthly_index = pd.date_range( + start=df.index.min(), + end=df.index.max() + pd.DateOffset(months=2), + freq="MS", + ) + interpolated = ( + df.reindex(df.index.union(monthly_index)) + .sort_index() + .interpolate(method="linear") + .ffill() + .bfill() + .reindex(monthly_index) + ) + converted = interpolated.iloc[::time_unit].copy() + return converted.iloc[:, 0] if is_series else converted + + +def prepare_labour_stats(country_name: Country, readers: DataReaders, time_unit: int = 3): labour_stats = readers.imf_reader.get_labour_stats(country_name) vacancy_rate = readers.oecd_econ.get_vacancy_rate(country_name) participation_rate = readers.world_bank.get_participation_rate(country_name) @@ -121,6 +192,12 @@ def prepare_labour_stats(country_name: Country, readers: DataReaders): }, inplace=True, ) + value_columns = [ + "Unemployment Rate (Value)", + "Participation Rate (Value)", + "Vacancy Rate (Value)", + ] + labour_stats = convert_levels_to_model_period(labour_stats[value_columns], time_unit) labour_stats["Unemployment Rate (Growth)"] = labour_stats["Unemployment Rate (Value)"].pct_change(fill_method=None) labour_stats["Participation Rate (Growth)"] = labour_stats["Participation Rate (Value)"].pct_change( fill_method=None @@ -208,7 +285,17 @@ def prepare_inflation(country_name: Country, readers: DataReaders): def normalised_growth(growth_rates: pd.Series, year: int, quarter: int): growth = (1 + growth_rates).cumprod() - return (growth / growth.loc[f"{year}-Q{quarter}"].values).values + base_date = pd.Timestamp(year, 3 * (quarter - 1) + 1, 1) + if base_date in growth.index: + base_value = growth.loc[base_date] + else: + base_value = ( + growth.reindex(growth.index.union(pd.DatetimeIndex([base_date]))) + .sort_index() + .interpolate(method="time") + .loc[base_date] + ) + return (growth / base_value).values def compile_national_accounts_data( diff --git a/macro_data/readers/population_data/hfcs_reader.py b/macro_data/readers/population_data/hfcs_reader.py index 2edca87a..4591b1f9 100644 --- a/macro_data/readers/population_data/hfcs_reader.py +++ b/macro_data/readers/population_data/hfcs_reader.py @@ -92,10 +92,23 @@ "DL1220": "Outstanding Balance of Credit Card Debt", # Credit card debt "DL1230": "Outstanding Balance of other Non-Mortgage Loans", # Other loans "HB0300": "Tenure Status of the Main Residence", # Housing tenure + "HB0410": "Rent Paid for Partially Owned Dwelling", # Monthly partial-ownership rent + "HB2001": "Mortgage Payment on Main Residence 1", # Monthly mortgage payment + "HB2002": "Mortgage Payment on Main Residence 2", # Monthly mortgage payment + "HB2003": "Mortgage Payment on Main Residence 3", # Monthly mortgage payment + "HB2200": "Additional Mortgage Payments on Main Residence", # Monthly mortgage payments "HB2300": "Rent Paid", # Rental payments "HB2410": "Number of Properties other than Household Main Residence", # Property count + "HB4200": "Other Property Loan Payments", # Monthly other-property loan payments + "HC1001": "Consumer Loan Payment 1", # Monthly consumer loan payment + "HC1002": "Consumer Loan Payment 2", # Monthly consumer loan payment + "HC1003": "Consumer Loan Payment 3", # Monthly consumer loan payment + "HC1200": "Additional Consumer Loan Payments", # Monthly consumer loan payments + "PF0930": "Pension Contributions", # Monthly pension contributions "DOCOGOODP": "Consumption of Consumer Goods/Services as a Share of Income", # Consumption ratio "HI0220": "Amount spent on Consumption of Goods and Services", # Total consumption + "HI0310": "Private Transfers Given", # Monthly private transfers + "PNF3610": "Health Insurance Payments", # Monthly health insurance payments } # List of variables containing monetary values that need currency conversion @@ -127,8 +140,21 @@ "Outstanding Balance of Credit Line", # Credit line "Outstanding Balance of Credit Card Debt", # Card debt "Outstanding Balance of other Non-Mortgage Loans", # Other debt + "Rent Paid for Partially Owned Dwelling", # Partial-ownership rent + "Mortgage Payment on Main Residence 1", # Mortgage payment + "Mortgage Payment on Main Residence 2", # Mortgage payment + "Mortgage Payment on Main Residence 3", # Mortgage payment + "Additional Mortgage Payments on Main Residence", # Mortgage payment "Rent Paid", # Rent expense + "Other Property Loan Payments", # Other-property loan payment + "Consumer Loan Payment 1", # Consumer loan payment + "Consumer Loan Payment 2", # Consumer loan payment + "Consumer Loan Payment 3", # Consumer loan payment + "Additional Consumer Loan Payments", # Consumer loan payment + "Pension Contributions", # Pension contribution "Amount spent on Consumption of Goods and Services", # Total spending + "Private Transfers Given", # Private transfer + "Health Insurance Payments", # Health insurance "Consumption of Consumer Goods/Services as a Share of Income", # Spending ratio ] diff --git a/macromodel/country/country.py b/macromodel/country/country.py index eacee03c..c267761d 100644 --- a/macromodel/country/country.py +++ b/macromodel/country/country.py @@ -344,6 +344,7 @@ def from_pickled_country( country_name=country_name, initial_year=initial_year, t_max=t_max, + time_unit=time_unit, ) economy = Economy.from_agents( diff --git a/macromodel/exogenous/exogenous.py b/macromodel/exogenous/exogenous.py index 4bf6fc27..f02f5098 100644 --- a/macromodel/exogenous/exogenous.py +++ b/macromodel/exogenous/exogenous.py @@ -36,6 +36,29 @@ from macromodel.exogenous.exogenous_ts import create_exogenous_timeseries +def _quarter_start_date(year: int, quarter: int) -> pd.Timestamp: + return pd.Timestamp(year, 3 * (quarter - 1) + 1, 1) + + +def _period_start_index(index: pd.Index, year: int, quarter: int) -> int: + date = _quarter_start_date(year, quarter) + index = pd.DatetimeIndex(index) + + exact_matches = np.where(index == date)[0] + if len(exact_matches) > 0: + return int(exact_matches[0]) + + prior_matches = np.where(index <= date)[0] + if len(prior_matches) > 0: + return int(prior_matches[-1]) + + later_matches = np.where(index > date)[0] + if len(later_matches) > 0: + return int(later_matches[0]) + + raise ValueError("Cannot select an initial period from an empty exogenous index.") + + class Exogenous: """External economic data manager. @@ -80,6 +103,7 @@ def __init__( vacancy_rate: pd.DataFrame, house_price_index: pd.DataFrame, exchange_rates_data: pd.DataFrame, + time_unit: int = 3, ): """Initialize exogenous data manager. @@ -106,30 +130,33 @@ def __init__( offset = 0 # Split data into before/during simulation periods - start_ind = np.where(self.inflation.index == str(initial_year) + "-Q" + str(initial_quarter))[0][0] + start_ind = _period_start_index(self.inflation.index, initial_year, initial_quarter) self.inflation_before = self.inflation.iloc[0:start_ind] self.inflation_during = self.inflation.iloc[start_ind : start_ind + t_max - offset] if len(self.national_accounts) > 0: + start_date = self.national_accounts.index[ + _period_start_index(self.national_accounts.index, initial_year, initial_quarter) + ] self.national_accounts_before = self.national_accounts.loc[ - self.national_accounts.index < pd.Timestamp(initial_year, 3 * initial_quarter - 2, 1) + self.national_accounts.index < start_date ] self.national_accounts_during = self.national_accounts.loc[ - self.national_accounts.index >= pd.Timestamp(initial_year, 3 * initial_quarter - 2, 1) + self.national_accounts.index >= start_date ] else: self.national_accounts_before = pd.DataFrame() self.national_accounts_during = pd.DataFrame() - start_ind = np.where(self.unemployment_rate.index == str(initial_year) + "-Q" + str(initial_quarter))[0][0] + start_ind = _period_start_index(self.unemployment_rate.index, initial_year, initial_quarter) self.unemployment_rate_before = self.unemployment_rate.iloc[0:start_ind] self.unemployment_rate_during = self.unemployment_rate.iloc[start_ind : start_ind + t_max - offset] - start_ind = np.where(self.vacancy_rate.index == str(initial_year) + "-Q" + str(initial_quarter))[0][0] + start_ind = _period_start_index(self.vacancy_rate.index, initial_year, initial_quarter) self.vacancy_rate_before = self.vacancy_rate.iloc[0:start_ind] self.vacancy_rate_during = self.vacancy_rate.iloc[start_ind : start_ind + t_max - offset] - start_ind = np.where(self.house_price_index.index == str(initial_year) + "-Q" + str(initial_quarter))[0][0] + start_ind = _period_start_index(self.house_price_index.index, initial_year, initial_quarter) self.house_price_index_before = self.house_price_index.iloc[0:start_ind] self.house_price_index_during = self.house_price_index.iloc[start_ind : start_ind + t_max - offset] @@ -143,11 +170,14 @@ def __init__( self.exchange_rates_data.index = [ind for ind in self.exchange_rates_data.index] self.exchange_rates_data.index = pd.PeriodIndex(self.exchange_rates_data.index, freq="Q").to_timestamp() self.exchange_rates_data.columns = ["Exchange Rate"] + exchange_rate_start_date = self.exchange_rates_data.index[ + _period_start_index(self.exchange_rates_data.index, initial_year, initial_quarter) + ] self.exchange_rates_data_before = self.exchange_rates_data.loc[ - self.exchange_rates_data.index < pd.Timestamp(initial_year, 3 * initial_quarter - 2, 1) + self.exchange_rates_data.index < exchange_rate_start_date ] self.exchange_rates_data_during = self.exchange_rates_data.loc[ - self.exchange_rates_data.index >= pd.Timestamp(initial_year, 3 * initial_quarter - 2, 1) + self.exchange_rates_data.index >= exchange_rate_start_date ] # Create time series and compile historic data @@ -158,6 +188,7 @@ def __init__( vacancy_rate_during=self.vacancy_rate_during, house_price_index_during=self.house_price_index_during, exchange_rates_data_during=self.exchange_rates_data_during, + time_unit=time_unit, ) self.compiled_historic_data = pd.concat( @@ -183,6 +214,7 @@ def from_pickled_agent( country_name: str, initial_year: int, t_max: int, + time_unit: int = 3, ): """Create instance from synthetic country data. @@ -211,6 +243,7 @@ def from_pickled_agent( initial_year=initial_year, initial_quarter=1, t_max=t_max, + time_unit=time_unit, ) def reset(self) -> None: diff --git a/macromodel/exogenous/exogenous_ts.py b/macromodel/exogenous/exogenous_ts.py index d72f8671..c7093985 100644 --- a/macromodel/exogenous/exogenous_ts.py +++ b/macromodel/exogenous/exogenous_ts.py @@ -33,6 +33,12 @@ from macromodel.timeseries import TimeSeries +def _periods_per_year(time_unit: int) -> int: + if time_unit <= 0 or 12 % time_unit != 0: + raise ValueError("time_unit must be a positive divisor of 12.") + return 12 // time_unit + + def create_exogenous_timeseries( inflation_during: pd.DataFrame, national_accounts_during: pd.DataFrame, @@ -40,6 +46,7 @@ def create_exogenous_timeseries( vacancy_rate_during: pd.DataFrame, house_price_index_during: pd.DataFrame, exchange_rates_data_during: pd.DataFrame, + time_unit: int = 3, ) -> TimeSeries: """Create a unified time series from exogenous economic data. @@ -163,10 +170,11 @@ def create_exogenous_timeseries( for t in range(1, len(national_accounts_during["Exports (Value)"].values) - offset): exog_ts.total_exports.append([national_accounts_during["Exports (Value)"].values[t]]) - # Update exchange rates with appropriate frequency - for t in range(1, len(exchange_rates_data_during.values)): - num = 4 if t > 1 else 3 - for _ in range(num): - exog_ts.exchange_rate.append([exchange_rates_data_during.values[t - 1]]) + # Exchange rates are annual; repeat each annual value over the configured model periods. + periods_per_year = _periods_per_year(time_unit) + for t, exchange_rate in enumerate(exchange_rates_data_during.values): + repeats = periods_per_year - 1 if t == 0 else periods_per_year + for _ in range(repeats): + exog_ts.exchange_rate.append([exchange_rate]) return exog_ts diff --git a/tests/test_macro_data/unit/test_readers/test_exogenous.py b/tests/test_macro_data/unit/test_readers/test_exogenous.py index 53104a20..93516c4c 100644 --- a/tests/test_macro_data/unit/test_readers/test_exogenous.py +++ b/tests/test_macro_data/unit/test_readers/test_exogenous.py @@ -1,5 +1,55 @@ +import numpy as np +import pandas as pd + from macro_data.configuration.countries import Country -from macro_data.readers.exogenous_data import ExogenousCountryData +from macro_data.readers.exogenous_data import ( + ExogenousCountryData, + convert_growth_rates_to_model_period, + convert_levels_to_model_period, +) + + +def test_convert_growth_rates_to_model_period_quarterly_noop(): + data = pd.DataFrame({"GDP": [0.03, 0.06]}, index=pd.to_datetime(["2020-01-01", "2020-04-01"])) + + converted = convert_growth_rates_to_model_period(data, time_unit=3) + + pd.testing.assert_frame_equal(converted, data) + + +def test_convert_growth_rates_to_model_period_monthly_compounds_to_quarterly_rate(): + data = pd.DataFrame({"GDP": [0.331]}, index=pd.to_datetime(["2020-01-01"])) + + converted = convert_growth_rates_to_model_period(data, time_unit=1) + + expected_monthly_rate = (1.331 ** (1.0 / 3.0)) - 1.0 + assert list(converted.index) == list(pd.to_datetime(["2020-01-01", "2020-02-01", "2020-03-01"])) + np.testing.assert_allclose(converted["GDP"].to_numpy(), expected_monthly_rate) + np.testing.assert_allclose((1.0 + converted["GDP"]).prod() - 1.0, 0.331) + + +def test_convert_growth_rates_to_model_period_annual_compounds_quarters(): + data = pd.DataFrame( + {"GDP": [0.1, 0.1, 0.1, 0.1]}, + index=pd.to_datetime(["2020-01-01", "2020-04-01", "2020-07-01", "2020-10-01"]), + ) + + converted = convert_growth_rates_to_model_period(data, time_unit=12) + + assert list(converted.index) == list(pd.to_datetime(["2020-01-01"])) + np.testing.assert_allclose(converted["GDP"].iloc[0], (1.1**4) - 1.0) + + +def test_convert_levels_to_model_period_interpolates_monthly_levels(): + data = pd.DataFrame({"Unemployment Rate (Value)": [0.03, 0.06]}, index=pd.to_datetime(["2020-01-01", "2020-04-01"])) + + converted = convert_levels_to_model_period(data, time_unit=1) + + expected = np.array([0.03, 0.04, 0.05, 0.06, 0.06, 0.06]) + assert list(converted.index) == list( + pd.to_datetime(["2020-01-01", "2020-02-01", "2020-03-01", "2020-04-01", "2020-05-01", "2020-06-01"]) + ) + np.testing.assert_allclose(converted["Unemployment Rate (Value)"].to_numpy(), expected) class TestExogenous: diff --git a/tests/test_macromodel/unit/test_exogenous/test_exogenous.py b/tests/test_macromodel/unit/test_exogenous/test_exogenous.py index e69de29b..662de6dd 100644 --- a/tests/test_macromodel/unit/test_exogenous/test_exogenous.py +++ b/tests/test_macromodel/unit/test_exogenous/test_exogenous.py @@ -0,0 +1,39 @@ +import numpy as np +import pandas as pd + +from macromodel.exogenous.exogenous_ts import create_exogenous_timeseries + + +def _minimal_exogenous_inputs(): + return { + "inflation_during": pd.DataFrame({"CPI Inflation": [0.01], "PPI Inflation": [0.02]}), + "national_accounts_during": pd.DataFrame(), + "unemployment_rate_during": pd.DataFrame({"Unemployment Rate (Value)": [0.05]}), + "vacancy_rate_during": pd.DataFrame({"Vacancy Rate (Value)": [0.01]}), + "house_price_index_during": pd.DataFrame( + { + "Real House Price Index Growth": [0.0], + "Nominal House Price Index Growth": [0.0], + } + ), + "exchange_rates_data_during": pd.DataFrame( + {"Exchange Rate": [1.2, 1.3]}, + index=pd.to_datetime(["2014-01-01", "2015-01-01"]), + ), + } + + +def _scalar_exchange_rates(exchange_rate_ts): + return [float(np.asarray(value).squeeze()) for value in exchange_rate_ts] + + +def test_create_exogenous_timeseries_repeats_annual_exchange_rates_quarterly(): + ts = create_exogenous_timeseries(**_minimal_exogenous_inputs(), time_unit=3) + + assert _scalar_exchange_rates(ts.exchange_rate) == [1.2] * 4 + [1.3] * 4 + + +def test_create_exogenous_timeseries_repeats_annual_exchange_rates_monthly(): + ts = create_exogenous_timeseries(**_minimal_exogenous_inputs(), time_unit=1) + + assert _scalar_exchange_rates(ts.exchange_rate) == [1.2] * 12 + [1.3] * 12 From 2be0432d1424469b72aa3439350b3bc76fdd5ab7 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 24 Apr 2026 19:25:45 +0100 Subject: [PATCH 2/9] Use configured period for Compustat readers --- .../population_data/compustat_banks_reader.py | 54 +++++- .../population_data/compustat_firms_reader.py | 52 ++++- .../test_readers/test_compustat_readers.py | 177 ++++++++++++++++++ 3 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 tests/test_macro_data/unit/test_readers/test_compustat_readers.py diff --git a/macro_data/readers/population_data/compustat_banks_reader.py b/macro_data/readers/population_data/compustat_banks_reader.py index a3f79692..2f27d075 100644 --- a/macro_data/readers/population_data/compustat_banks_reader.py +++ b/macro_data/readers/population_data/compustat_banks_reader.py @@ -74,6 +74,21 @@ "dltry": "Long-term Debt Reduction", # Debt repayments } +quarterly_stock_source_variables = [ + "atq", # Total assets + "dlttq", # Long-term debt + "dptcq", # Customer deposits + "ltq", # Total liabilities + "teqq", # Total equity +] +quarterly_flow_source_variables = [ + "ciq", # Total income + "dltisy", # New debt issued + "dltry", # Debt repayments +] +quarterly_stock_variables = [var_mapping[column] for column in quarterly_stock_source_variables] +quarterly_flow_variables = [var_mapping[column] for column in quarterly_flow_source_variables] + # List of variables containing monetary values var_numerical = [ "Assets", # Total assets @@ -96,6 +111,32 @@ ] +def _active_quarterly_flow_source_columns() -> list[str]: + return [ + source_column + for source_column, mapped_column in var_mapping.items() + if source_column in quarterly_flow_source_variables and mapped_column in var_keeping + ] + + +def _validate_time_unit(time_unit: int) -> None: + if time_unit <= 0 or 12 % time_unit != 0: + raise ValueError("Compustat flow conversion requires `time_unit` to be a positive divisor of 12.") + + +def _convert_active_quarterly_flows_to_time_unit( + data: pd.DataFrame, active_flow_columns: list[str], time_unit: int +) -> pd.DataFrame: + if not active_flow_columns: + return data + + _validate_time_unit(time_unit) + conversion_factor = time_unit / 3 + if conversion_factor != 1: + data.loc[:, active_flow_columns] = data[active_flow_columns].astype(float) * conversion_factor + return data + + class CompustatBanksReader: """ A class for reading and processing Compustat bank-level financial data. @@ -138,6 +179,7 @@ def from_raw_data( raw_quarterly_path: Path | str, countries: list[str | Country], proxy_with_us: bool = True, + time_unit: int = 3, ): """ Create a CompustatBanksReader instance from raw Compustat data. @@ -160,6 +202,9 @@ def from_raw_data( List of countries to include in the data proxy_with_us : bool, optional Whether to include US banks for proxying (default: True) + time_unit : int, optional + Target simulation period length in months. Active quarterly flow fields + are scaled linearly from quarterly totals to this period. Returns ------- @@ -176,7 +221,12 @@ def from_raw_data( raw_data = pd.read_csv(raw_quarterly_path, encoding="unicode_escape", engine="pyarrow") # Filter for time period - data = raw_data[np.logical_and(raw_data["fyearq"] == year, raw_data["fqtr"] == quarter)] + data = raw_data[np.logical_and(raw_data["fyearq"] == year, raw_data["fqtr"] == quarter)].copy() + data = _convert_active_quarterly_flows_to_time_unit( + data=data, + active_flow_columns=_active_quarterly_flow_source_columns(), + time_unit=time_unit, + ) # Add US banks if needed for proxying if proxy_with_us: @@ -195,7 +245,7 @@ def from_raw_data( # Impute missing values by country for c in data.index.get_level_values(0).unique(): data_values = data.loc[c].values - if len(data_values) == 1: + if data_values.ndim == 1: data_values = data_values.reshape(1, -1) data.loc[c] = IterativeImputer().fit_transform(data_values) diff --git a/macro_data/readers/population_data/compustat_firms_reader.py b/macro_data/readers/population_data/compustat_firms_reader.py index 398c21ba..96478dd6 100644 --- a/macro_data/readers/population_data/compustat_firms_reader.py +++ b/macro_data/readers/population_data/compustat_firms_reader.py @@ -73,6 +73,21 @@ "loc": "Country", # Country location } +quarterly_stock_source_variables = [ + "atq", # Total assets + "ceqq", # Common equity + "dlttq", # Long-term debt + "dptbq", # Bank deposits + "invtq", # Total inventory + "ltq", # Total liabilities +] +quarterly_flow_source_variables = [ + "revtq", # Total revenue + "gpq", # Gross profits +] +quarterly_stock_variables = [var_mapping[column] for column in quarterly_stock_source_variables] +quarterly_flow_variables = [var_mapping[column] for column in quarterly_flow_source_variables] + # List of variables containing monetary values var_numerical = [ "Assets", # Total assets @@ -105,6 +120,32 @@ simplefilter("ignore", category=ConvergenceWarning) +def _active_quarterly_flow_source_columns() -> list[str]: + return [ + source_column + for source_column, mapped_column in var_mapping.items() + if source_column in quarterly_flow_source_variables and mapped_column in var_keeping + ] + + +def _validate_time_unit(time_unit: int) -> None: + if time_unit <= 0 or 12 % time_unit != 0: + raise ValueError("Compustat flow conversion requires `time_unit` to be a positive divisor of 12.") + + +def _convert_active_quarterly_flows_to_time_unit( + data: pd.DataFrame, active_flow_columns: list[str], time_unit: int +) -> pd.DataFrame: + if not active_flow_columns: + return data + + _validate_time_unit(time_unit) + conversion_factor = time_unit / 3 + if conversion_factor != 1: + data.loc[:, active_flow_columns] = data[active_flow_columns].astype(float) * conversion_factor + return data + + class CompustatFirmsReader: """ A class for reading and processing Compustat firm-level financial data. @@ -147,6 +188,7 @@ def from_raw_data( raw_annual_path: Path | str, raw_quarterly_path: Path | str, countries: list[str | Country], + time_unit: int = 3, ): """ Create a CompustatFirmsReader instance from raw Compustat files. @@ -170,6 +212,9 @@ def from_raw_data( Path to quarterly Compustat data file countries : list[str | Country] List of countries to include in the data + time_unit : int, optional + Target simulation period length in months. Active quarterly flow fields + are scaled linearly from quarterly totals to this period. Returns ------- @@ -197,7 +242,12 @@ def from_raw_data( raw_quarterly_data["fyearq"] == year, raw_quarterly_data["fqtr"] == quarter, ) - ] + ].copy() + raw_quarterly_data = _convert_active_quarterly_flows_to_time_unit( + data=raw_quarterly_data, + active_flow_columns=_active_quarterly_flow_source_columns(), + time_unit=time_unit, + ) # Clean and filter data annual_data = raw_annual_data.dropna(axis=0, how="all").dropna(axis=1, how="all") diff --git a/tests/test_macro_data/unit/test_readers/test_compustat_readers.py b/tests/test_macro_data/unit/test_readers/test_compustat_readers.py new file mode 100644 index 00000000..7ccb32c5 --- /dev/null +++ b/tests/test_macro_data/unit/test_readers/test_compustat_readers.py @@ -0,0 +1,177 @@ +import pandas as pd +import pytest + +from macro_data.configuration.countries import Country +from macro_data.readers.population_data.compustat_banks_reader import CompustatBanksReader +from macro_data.readers.population_data.compustat_firms_reader import CompustatFirmsReader + + +def _write_firm_files(tmp_path): + annual_path = tmp_path / "firms_annual.csv" + quarterly_path = tmp_path / "firms_quarterly.csv" + + pd.DataFrame( + [ + { + "fyear": 2014, + "datadate": "2014-12-31", + "emp": 10.0, + "conm": "FIRM A", + "loc": "USA", + } + ] + ).to_csv(annual_path, index=False) + + pd.DataFrame( + [ + { + "curcdq": "USD", + "fqtr": 1, + "fyearq": 2014, + "datadate": "2014-03-31", + "atq": 100.0, + "ceqq": 50.0, + "dlttq": 20.0, + "dptbq": 15.0, + "gpq": 30.0, + "invtq": 5.0, + "ltq": 60.0, + "revtq": 90.0, + "conm": "FIRM A", + "gsector": 10.0, + "loc": "USA", + }, + { + "curcdq": "USD", + "fqtr": 2, + "fyearq": 2014, + "datadate": "2014-06-30", + "atq": 200.0, + "ceqq": 70.0, + "dlttq": 40.0, + "dptbq": 25.0, + "gpq": 60.0, + "invtq": 10.0, + "ltq": 80.0, + "revtq": 120.0, + "conm": "FIRM A", + "gsector": 10.0, + "loc": "USA", + }, + ] + ).to_csv(quarterly_path, index=False) + + return annual_path, quarterly_path + + +def _write_bank_file(tmp_path): + path = tmp_path / "banks.csv" + pd.DataFrame( + [ + { + "curcdq": "USD", + "fqtr": 1, + "fyearq": 2014, + "datadate": "2014-03-31", + "conm": "BANK A", + "atq": 100.0, + "ciq": 9.0, + "dlttq": 20.0, + "dptcq": 15.0, + "ltq": 60.0, + "teqq": 50.0, + "dltisy": 3.0, + "dltry": 2.0, + "loc": "USA", + }, + { + "curcdq": "USD", + "fqtr": 2, + "fyearq": 2014, + "datadate": "2014-06-30", + "conm": "BANK A", + "atq": 200.0, + "ciq": 12.0, + "dlttq": 40.0, + "dptcq": 25.0, + "ltq": 80.0, + "teqq": 70.0, + "dltisy": 6.0, + "dltry": 4.0, + "loc": "USA", + }, + ] + ).to_csv(path, index=False) + return path + + +def test__compustat_firms_uses_configured_quarter(tmp_path): + annual_path, quarterly_path = _write_firm_files(tmp_path) + + reader = CompustatFirmsReader.from_raw_data( + year=2014, + quarter=2, + raw_annual_path=annual_path, + raw_quarterly_path=quarterly_path, + countries=[Country("USA")], + ) + + firm = reader.data.iloc[0] + assert firm["Assets"] == pytest.approx(200.0) + assert firm["Revenue"] == pytest.approx(120.0) + assert firm["Profits"] == pytest.approx(60.0) + + +def test__compustat_firms_converts_active_quarterly_flows_to_monthly(tmp_path): + annual_path, quarterly_path = _write_firm_files(tmp_path) + + reader = CompustatFirmsReader.from_raw_data( + year=2014, + quarter=2, + raw_annual_path=annual_path, + raw_quarterly_path=quarterly_path, + countries=[Country("USA")], + time_unit=1, + ) + + firm = reader.data.iloc[0] + assert firm["Assets"] == pytest.approx(200.0) + assert firm["Revenue"] == pytest.approx(40.0) + assert firm["Profits"] == pytest.approx(20.0) + + +def test__compustat_firms_converts_active_quarterly_flows_to_bimonthly(tmp_path): + annual_path, quarterly_path = _write_firm_files(tmp_path) + + reader = CompustatFirmsReader.from_raw_data( + year=2014, + quarter=2, + raw_annual_path=annual_path, + raw_quarterly_path=quarterly_path, + countries=[Country("USA")], + time_unit=2, + ) + + firm = reader.data.iloc[0] + assert firm["Revenue"] == pytest.approx(80.0) + assert firm["Profits"] == pytest.approx(40.0) + + +def test__compustat_banks_uses_configured_quarter_without_converting_inactive_flows(tmp_path): + bank_path = _write_bank_file(tmp_path) + + reader = CompustatBanksReader.from_raw_data( + year=2014, + quarter=2, + raw_quarterly_path=bank_path, + countries=[Country("USA")], + proxy_with_us=False, + time_unit=1, + ) + + bank = reader.data.iloc[0] + assert "Income" not in reader.data.columns + assert "Long-term Debt Issuance" not in reader.data.columns + assert "Long-term Debt Reduction" not in reader.data.columns + assert bank["Assets"] == pytest.approx(200.0) + assert bank["Debt"] == pytest.approx(40.0) From a83a85ebabbe74ad83398a43fed9ed410d2560cb Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 24 Apr 2026 19:25:54 +0100 Subject: [PATCH 3/9] Source data loan maturities from model defaults --- macro_data/configuration/dataconfiguration.py | 32 +++++++++++++------ macro_data/default_country_conf.yaml | 10 ++---- .../configurations/bank_configuration.py | 32 ++++++++++++++----- .../test_macro_data/unit/data_config_gen.yaml | 4 +-- .../unit/default_data_config.yaml | 2 +- .../unit/default_unit_test.yaml | 15 +++++---- .../unit/default_unit_test.yaml | 11 ++++--- 7 files changed, 67 insertions(+), 39 deletions(-) diff --git a/macro_data/configuration/dataconfiguration.py b/macro_data/configuration/dataconfiguration.py index 1f401cd1..94b49804 100644 --- a/macro_data/configuration/dataconfiguration.py +++ b/macro_data/configuration/dataconfiguration.py @@ -23,10 +23,7 @@ initial_inventory_to_input_fraction=0.1 ) - banks_config = BanksDataConfiguration( - long_term_firm_loan_maturity=60, - mortgage_maturity=120 - ) + banks_config = BanksDataConfiguration() # Create country configuration france_config = CountryDataConfiguration( @@ -54,9 +51,13 @@ from pydantic import BaseModel, Field +from macromodel.configurations.bank_configuration import BankParameters + from .countries import Country from .region import Region +_MODEL_BANK_PARAMETERS = BankParameters() + class FirmsDataConfiguration(BaseModel): """ @@ -107,18 +108,29 @@ class BanksDataConfiguration(BaseModel): This class controls how synthetic bank data is generated, including loan maturities and interest rate settings. + Loan maturities are expressed as model-period counts, not calendar months. + Attributes: constructor (Literal["Compustat", "Default"]): The data constructor to use - long_term_firm_loan_maturity (int): Maturity period (months) for long-term firm loans - consumption_exp_loan_maturity (int): Maturity period (months) for consumption loans - mortgage_maturity (int): Maturity period (months) for mortgages + long_term_firm_loan_maturity (int): Initial maturity in model periods, not months, for long-term firm loans + consumption_exp_loan_maturity (int): Initial maturity in model periods, not months, for consumption loans + mortgage_maturity (int): Initial maturity in model periods, not months, for mortgages interest_rates (InterestRates): Interest rate markup configuration """ constructor: Literal["Compustat", "Default"] = "Compustat" - long_term_firm_loan_maturity: int = 60 - consumption_exp_loan_maturity: int = 12 - mortgage_maturity: int = 120 + long_term_firm_loan_maturity: int = Field( + _MODEL_BANK_PARAMETERS.long_term_firm_loan_maturity, + description="Initial maturity in model periods, not months, for long-term firm loans.", + ) + consumption_exp_loan_maturity: int = Field( + _MODEL_BANK_PARAMETERS.household_consumption_loan_maturity, + description="Initial maturity in model periods, not months, for consumption loans.", + ) + mortgage_maturity: int = Field( + _MODEL_BANK_PARAMETERS.mortgage_maturity, + description="Initial maturity in model periods, not months, for mortgages.", + ) interest_rates: InterestRates = InterestRates() diff --git a/macro_data/default_country_conf.yaml b/macro_data/default_country_conf.yaml index c30a473c..c7941120 100644 --- a/macro_data/default_country_conf.yaml +++ b/macro_data/default_country_conf.yaml @@ -17,17 +17,11 @@ central_bank_configuration: 0.02 banks_configuration: - long_term_firm_loan_maturity: - 60 - consumption_exp_loan_maturity: - 12 - mortgage_maturity: - 120 - + # Loan maturities are expressed in model periods, not calendar months. interest_rates: consumption_loans_markup: 0.01 mortgage_markup: 0.1 household_overdraft_markup: - 0.01 \ No newline at end of file + 0.01 diff --git a/macromodel/configurations/bank_configuration.py b/macromodel/configurations/bank_configuration.py index c0e27cd2..79658d1e 100644 --- a/macromodel/configurations/bank_configuration.py +++ b/macromodel/configurations/bank_configuration.py @@ -27,10 +27,10 @@ class BankParameters(BaseModel): mortgage_loan_to_income_ratio (float): Maximum mortgage to income ratio mortgage_loan_to_value_ratio (float): Maximum mortgage to property value mortgage_debt_service_to_income_ratio (float): Maximum mortgage payment to income - household_consumption_loan_maturity (int): Consumer loan term in periods - long_term_firm_loan_maturity (int): Long-term firm loan term in months - mortgage_maturity (int): Mortgage term in months - short_term_firm_loan_maturity (int): Short-term firm loan term in months + household_consumption_loan_maturity (int): Consumer loan term in model periods + long_term_firm_loan_maturity (int): Long-term firm loan term in model periods + mortgage_maturity (int): Mortgage term in model periods + short_term_firm_loan_maturity (int): Short-term firm loan term in model periods """ capital_adequacy_ratio: float = Field(ge=0, le=1, default=0.08) @@ -41,10 +41,26 @@ class BankParameters(BaseModel): mortgage_loan_to_income_ratio: float = Field(ge=0, le=1, default=0.05) mortgage_loan_to_value_ratio: float = Field(ge=0, le=1, default=0.05) mortgage_debt_service_to_income_ratio: float = Field(ge=0, le=1, default=0.05) - household_consumption_loan_maturity: int = Field(ge=0, default=1) - long_term_firm_loan_maturity: int = Field(ge=0, default=60) - mortgage_maturity: int = Field(ge=0, default=120) - short_term_firm_loan_maturity: int = Field(ge=0, default=20) + household_consumption_loan_maturity: int = Field( + ge=0, + default=1, + description="Consumer loan term in model periods.", + ) + long_term_firm_loan_maturity: int = Field( + ge=0, + default=60, + description="Long-term firm loan term in model periods.", + ) + mortgage_maturity: int = Field( + ge=0, + default=120, + description="Mortgage term in model periods.", + ) + short_term_firm_loan_maturity: int = Field( + ge=0, + default=20, + description="Short-term firm loan term in model periods.", + ) class DemographyFunction(BaseModel): diff --git a/tests/test_macro_data/unit/data_config_gen.yaml b/tests/test_macro_data/unit/data_config_gen.yaml index 5bcebe59..ad591498 100644 --- a/tests/test_macro_data/unit/data_config_gen.yaml +++ b/tests/test_macro_data/unit/data_config_gen.yaml @@ -49,7 +49,7 @@ country_configs: long_term_firm_loan_maturity: 60 consumption_exp_loan_maturity: - 12 + 1 mortgage_maturity: 120 @@ -86,7 +86,7 @@ country_configs: long_term_firm_loan_maturity: 60 consumption_exp_loan_maturity: - 12 + 1 mortgage_maturity: 120 diff --git a/tests/test_macro_data/unit/default_data_config.yaml b/tests/test_macro_data/unit/default_data_config.yaml index e0794a0c..af0ebd37 100644 --- a/tests/test_macro_data/unit/default_data_config.yaml +++ b/tests/test_macro_data/unit/default_data_config.yaml @@ -40,7 +40,7 @@ country_configs: long_term_firm_loan_maturity: 60 consumption_exp_loan_maturity: - 12 + 1 mortgage_maturity: 120 diff --git a/tests/test_macro_data/unit/default_unit_test.yaml b/tests/test_macro_data/unit/default_unit_test.yaml index b6172201..42a1d977 100644 --- a/tests/test_macro_data/unit/default_unit_test.yaml +++ b/tests/test_macro_data/unit/default_unit_test.yaml @@ -723,13 +723,14 @@ init: banks: parameters: household_consumption_expansion_loan_maturity: - desc: The initial loan maturity in months for household consumption expansion - loans. + desc: The initial loan maturity in model periods, not months, for household + consumption expansion loans. options: '' type: int value: 12 household_payday_loan_maturity: - desc: The initial loan maturity in months for household payday loans. + desc: The initial loan maturity in model periods, not months, for household + payday loans. options: '' type: int value: 1 @@ -752,17 +753,19 @@ init: type: float value: 0.1 long_term_firm_loan_maturity: - desc: The initial loan maturity in months for long-term firm loans. + desc: The initial loan maturity in model periods, not months, for long-term + firm loans. options: '' type: int value: 60 mortgage_maturity: - desc: The initial loan maturity in months for mortgages. + desc: The initial loan maturity in model periods, not months, for mortgages. options: '' type: int value: 120 short_term_firm_loan_maturity: - desc: The initial loan maturity in months for short-term firm loans. + desc: The initial loan maturity in model periods, not months, for short-term + firm loans. options: '' type: int value: 12 diff --git a/tests/test_macromodel/unit/default_unit_test.yaml b/tests/test_macromodel/unit/default_unit_test.yaml index 779eb15a..cf3e2ab3 100644 --- a/tests/test_macromodel/unit/default_unit_test.yaml +++ b/tests/test_macromodel/unit/default_unit_test.yaml @@ -1056,22 +1056,25 @@ init: type: str value: compustat_synthetic_banks.CompustatSyntheticBanks household_consumption_loan_maturity: - desc: The initial loan maturity in months for household consumption loans. + desc: The initial loan maturity in model periods, not months, for household + consumption loans. options: '' type: int value: 1 long_term_firm_loan_maturity: - desc: The initial loan maturity in months for long-term firm loans. + desc: The initial loan maturity in model periods, not months, for long-term + firm loans. options: '' type: int value: 60 mortgage_maturity: - desc: The initial loan maturity in months for mortgages. + desc: The initial loan maturity in model periods, not months, for mortgages. options: '' type: int value: 120 short_term_firm_loan_maturity: - desc: The initial loan maturity in months for short-term firm loans. + desc: The initial loan maturity in model periods, not months, for short-term + firm loans. options: '' type: int value: 20 From ebb3f64ad6735cf96c1a24145bd3f78c64866f87 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 24 Apr 2026 19:26:02 +0100 Subject: [PATCH 4/9] Track initial credit totals from principal balances --- .../markets/credit_market/credit_market.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/macromodel/markets/credit_market/credit_market.py b/macromodel/markets/credit_market/credit_market.py index 45297c7a..06c96bb0 100644 --- a/macromodel/markets/credit_market/credit_market.py +++ b/macromodel/markets/credit_market/credit_market.py @@ -147,10 +147,10 @@ def from_pickled_market( mortgage_loans = synthetic_credit_market.mortgage_loans.stack() ts = create_credit_market_timeseries( - total_consumption_expansion_loans=consumption_expansion_loans.sum(), - total_short_term_loans=shortterm_loans.sum(), - total_long_term_loans=longterm_loans.sum(), - total_mortgage_loans=mortgage_loans.sum(), + total_consumption_expansion_loans=consumption_expansion_loans[0].sum(), + total_short_term_loans=shortterm_loans[0].sum(), + total_long_term_loans=longterm_loans[0].sum(), + total_mortgage_loans=mortgage_loans[0].sum(), ) states = { @@ -225,10 +225,10 @@ def from_data( # Create the corresponding time series object ts = create_credit_market_timeseries( - total_short_term_loans=st_loans.sum(), - total_long_term_loans=lt_loans.sum(), - total_consumption_expansion_loans=cons_loans.sum(), - total_mortgage_loans=mort_loans.sum(), + total_short_term_loans=st_loans[0].sum(), + total_long_term_loans=lt_loans[0].sum(), + total_consumption_expansion_loans=cons_loans[0].sum(), + total_mortgage_loans=mort_loans[0].sum(), ) return cls( From cfba6c57156a6f291b2120ec22f173b80ce2c6e5 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 15 May 2026 22:15:29 +0100 Subject: [PATCH 5/9] style: apply ruff formatting --- macro_data/readers/default_readers.py | 6 +++--- macromodel/exogenous/exogenous.py | 8 ++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/macro_data/readers/default_readers.py b/macro_data/readers/default_readers.py index 277255aa..33c28c62 100644 --- a/macro_data/readers/default_readers.py +++ b/macro_data/readers/default_readers.py @@ -628,9 +628,9 @@ def get_total_benefits_lcu(self, country_name: Country, year: int, yearly_factor Returns: float: Total benefits in local currency units """ - return self.oecd_econ.all_benefits_gdp_pct( - country_name, year - ) * self.world_bank.get_current_scaled_gdp(country_name, year, rescale_factor=yearly_factor) + return self.oecd_econ.all_benefits_gdp_pct(country_name, year) * self.world_bank.get_current_scaled_gdp( + country_name, year, rescale_factor=yearly_factor + ) def get_total_unemployment_benefits_lcu( self, country_name: Country, year: int, yearly_factor: float = 4.0 diff --git a/macromodel/exogenous/exogenous.py b/macromodel/exogenous/exogenous.py index f02f5098..17341da3 100644 --- a/macromodel/exogenous/exogenous.py +++ b/macromodel/exogenous/exogenous.py @@ -138,12 +138,8 @@ def __init__( start_date = self.national_accounts.index[ _period_start_index(self.national_accounts.index, initial_year, initial_quarter) ] - self.national_accounts_before = self.national_accounts.loc[ - self.national_accounts.index < start_date - ] - self.national_accounts_during = self.national_accounts.loc[ - self.national_accounts.index >= start_date - ] + self.national_accounts_before = self.national_accounts.loc[self.national_accounts.index < start_date] + self.national_accounts_during = self.national_accounts.loc[self.national_accounts.index >= start_date] else: self.national_accounts_before = pd.DataFrame() self.national_accounts_during = pd.DataFrame() From 50a2c49a6e095d2e68f5c9ec596feede111a8ef6 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 15 May 2026 22:18:55 +0100 Subject: [PATCH 6/9] fix: align data wrapper emissions arg with main --- macro_data/data_wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/macro_data/data_wrapper.py b/macro_data/data_wrapper.py index 52990988..ca017492 100644 --- a/macro_data/data_wrapper.py +++ b/macro_data/data_wrapper.py @@ -246,7 +246,6 @@ def from_config( use_disagg_can_2014_reader=configuration.can_disaggregation, use_provincial_can_reader=use_provincial_can_reader, regions_dict=regions_dict, - allow_missing_emissions=allow_missing_emissions, yearly_factor=yearly_factor, simulation_quarter=quarter, ) From d817ccd42b6ffbccfe4384c2bf25a013b002fa78 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 15 May 2026 22:22:45 +0100 Subject: [PATCH 7/9] fix: wire time_unit through country and synthetic builders --- macro_data/data_wrapper.py | 4 +++- macro_data/processing/synthetic_country.py | 2 ++ macromodel/country/country.py | 1 + macromodel/simulation.py | 1 + tests/test_macromodel/unit/test_country/test_country.py | 1 + 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/macro_data/data_wrapper.py b/macro_data/data_wrapper.py index ca017492..b5555be2 100644 --- a/macro_data/data_wrapper.py +++ b/macro_data/data_wrapper.py @@ -331,6 +331,7 @@ def from_config( country=country, year=year, quarter=quarter, + time_unit=configuration.time_unit, country_configuration=configuration.country_configs[country], industries=industries, readers=readers, @@ -356,6 +357,8 @@ def from_config( country=country, proxy_country=configuration.country_configs[country].eu_proxy_country, year=year, + quarter=quarter, + time_unit=configuration.time_unit, country_configuration=configuration.country_configs[country], industries=industries, readers=readers, @@ -363,7 +366,6 @@ def from_config( country_industry_data=industry_data[country], year_range=year_range, goods_criticality_matrix=readers.goods_criticality.criticality_matrix, - quarter=quarter, proxy_inflation_data=proxy_inflation[country], emission_factors=( EmissionsData.from_readers( diff --git a/macro_data/processing/synthetic_country.py b/macro_data/processing/synthetic_country.py index 1fa7524e..fd1e9d17 100644 --- a/macro_data/processing/synthetic_country.py +++ b/macro_data/processing/synthetic_country.py @@ -176,6 +176,7 @@ def eu_synthetic_country( country: Country, year: int, quarter: int, + time_unit: int, country_configuration: CountryDataConfiguration, industries: list[str], readers: DataReaders, @@ -363,6 +364,7 @@ def proxied_synthetic_country( proxy_country: Country, year: int, quarter: int, + time_unit: int, country_configuration: CountryDataConfiguration, industries: list[str], readers: DataReaders, diff --git a/macromodel/country/country.py b/macromodel/country/country.py index c267761d..a29302bb 100644 --- a/macromodel/country/country.py +++ b/macromodel/country/country.py @@ -220,6 +220,7 @@ def from_pickled_country( initial_year: int, t_max: int, running_multiple_countries: bool, + time_unit: int, emission_factors_usd: np.ndarray, ) -> "Country": """Create a Country instance from preprocessed synthetic data. diff --git a/macromodel/simulation.py b/macromodel/simulation.py index b8977771..691d586f 100644 --- a/macromodel/simulation.py +++ b/macromodel/simulation.py @@ -142,6 +142,7 @@ def from_datawrapper( initial_year=datawrapper.configuration.year, t_max=simulation_configuration.t_max, running_multiple_countries=running_multi_country, + time_unit=datawrapper.configuration.time_unit, emission_factors_usd=emission_factors, ) for country_name in countries_without_row diff --git a/tests/test_macromodel/unit/test_country/test_country.py b/tests/test_macromodel/unit/test_country/test_country.py index a48b8987..8d02bd77 100644 --- a/tests/test_macromodel/unit/test_country/test_country.py +++ b/tests/test_macromodel/unit/test_country/test_country.py @@ -40,6 +40,7 @@ def test__init(self, datawrapper): initial_year=datawrapper.configuration.year, t_max=12, running_multiple_countries=False, + time_unit=datawrapper.configuration.time_unit, emission_factors_usd=emission_factors, ) From 57e3ded7f66fded535ff01cf6651332c4dc757d2 Mon Sep 17 00:00:00 2001 From: agurgone Date: Fri, 15 May 2026 22:32:16 +0100 Subject: [PATCH 8/9] fix: fallback time_unit for legacy datawrapper configs --- macromodel/simulation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macromodel/simulation.py b/macromodel/simulation.py index 691d586f..e75b95d9 100644 --- a/macromodel/simulation.py +++ b/macromodel/simulation.py @@ -106,6 +106,7 @@ def from_datawrapper( countries_with_row = datawrapper.all_country_names running_multi_country = len(countries_without_row) > 1 + time_unit = int(getattr(datawrapper.configuration, "time_unit", datawrapper.time_unit)) model_dict = { country_name: country.synthetic_goods_market.exchange_rates_model @@ -142,7 +143,7 @@ def from_datawrapper( initial_year=datawrapper.configuration.year, t_max=simulation_configuration.t_max, running_multiple_countries=running_multi_country, - time_unit=datawrapper.configuration.time_unit, + time_unit=time_unit, emission_factors_usd=emission_factors, ) for country_name in countries_without_row From 81f413465002dd32c54913bd721ac35855c4d69f Mon Sep 17 00:00:00 2001 From: agurgone Date: Sat, 16 May 2026 19:51:34 +0100 Subject: [PATCH 9/9] fix: preserve quarterly default when config omits time_unit --- macro_data/data_wrapper.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/macro_data/data_wrapper.py b/macro_data/data_wrapper.py index b5555be2..5017c0b8 100644 --- a/macro_data/data_wrapper.py +++ b/macro_data/data_wrapper.py @@ -215,7 +215,10 @@ def from_config( year = configuration.year quarter = configuration.quarter - yearly_factor = 12 / configuration.time_unit + # Backward compatibility: legacy configs omitted `time_unit` and implicitly used quarterly scaling. + fields_set = getattr(configuration, "model_fields_set", set()) + effective_time_unit = configuration.time_unit if "time_unit" in fields_set else 3 + yearly_factor = 12 / effective_time_unit scale_dict = {country: configuration.country_configs[country].scale for country in country_names} @@ -293,7 +296,7 @@ def from_config( quarter=quarter, industry_vectors=industry_data[country]["industry_vectors"], proxy_country=proxy_country_dict.get(country, None), - time_unit=configuration.time_unit, + time_unit=effective_time_unit, ) for country in country_names } @@ -308,7 +311,7 @@ def from_config( inflation = readers.imf_reader.get_inflation(proxy_country) if inflation is None: inflation = readers.world_bank.get_inflation(proxy_country) - inflation = convert_growth_rates_to_model_period(inflation, configuration.time_unit) + inflation = convert_growth_rates_to_model_period(inflation, effective_time_unit) proxy_inflation[country] = inflation else: proxy_inflation[country] = None @@ -331,7 +334,7 @@ def from_config( country=country, year=year, quarter=quarter, - time_unit=configuration.time_unit, + time_unit=effective_time_unit, country_configuration=configuration.country_configs[country], industries=industries, readers=readers, @@ -358,7 +361,7 @@ def from_config( proxy_country=configuration.country_configs[country].eu_proxy_country, year=year, quarter=quarter, - time_unit=configuration.time_unit, + time_unit=effective_time_unit, country_configuration=configuration.country_configs[country], industries=industries, readers=readers, @@ -414,7 +417,7 @@ def from_config( EmissionsEnergyFactors.from_readers(readers.icio[year], country_names) if add_emissions else None ), aggregation_structure=configuration.aggregation_structure, - time_unit=configuration.time_unit, + time_unit=effective_time_unit, ) @classmethod