Skip to content

Update code/new data #226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dav-sonn wants to merge 3 commits into from
Closed

Update code/new data #226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c4367bf
Code changes to accommodate new data
dav-sonn Aug 6, 2025
4aaa5ee
Update of the code with the latests estimates
dav-sonn Aug 28, 2025
3bdeba0
Update persistence.xml
dav-sonn Sep 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config/default.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ executeWithGui: false
randomSeed: 606
startYear: 2019
endYear: 2026
popSize: 170000
popSize: 50000

# Arguments passed to the SimPathsModel
model_args:
Expand Down
Binary file modified input/DatabaseCountryYear.xlsx
View file
Open in desktop
Binary file not shown.
Binary file added input/EUROMODoutput/DatabaseCountryYear.xlsx
View file
Open in desktop
Binary file not shown.
Binary file added input/EUROMODoutput/EUROMODpolicySchedule.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/EUROMODpolicySchedule.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_RMSE.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_education.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_employmentSelection.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_fertility.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_health.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_home_ownership.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_income.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_leaveParentalHome.xlsx
View file
Open in desktop
Binary file not shown.
Binary file added input/reg_leave_parental_home.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_partnership.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_retirement.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/reg_wages.xlsx
View file
Open in desktop
Binary file not shown.
Binary file modified input/scenario_parametricMatching.xlsx
View file
Open in desktop
Binary file not shown.
18 changes: 9 additions & 9 deletions src/main/java/simpaths/data/ManagerRegressions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,15 +196,6 @@ public static OrderedRegression getOrderedRegression(RegressionName regression)
throw new RuntimeException("requested ordered regression is not recognised: " + regression.name());

switch (regression) {
case HealthH1a -> {
return Parameters.getRegHealthH1a();
}
case HealthH1b -> {
return Parameters.getRegHealthH1b();
}
case EducationE2a -> {
return Parameters.getRegEducationE2a();
}
default -> {
throw new RuntimeException("unrecognised regression (1)");
}
Expand All @@ -217,6 +208,15 @@ public static GeneralisedOrderedRegression getGeneralisedOrderedRegression(Regre
throw new RuntimeException("requested generalised ordered regression is not recognised: " + regression.name());

switch (regression) {
case HealthH1a -> {
return Parameters.getRegHealthH1a();
}
case HealthH1b -> {
return Parameters.getRegHealthH1b();
}
case EducationE2a -> {
return Parameters.getRegEducationE2a();
}
default -> {
throw new RuntimeException("unrecognised regression (1)");
}
Expand Down
103 changes: 53 additions & 50 deletions src/main/java/simpaths/data/Parameters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,16 @@ public class Parameters {
"ded", //in education dummy
"der", //return to education dummy
"dot", //ethnicity
"dot01", //ethnicity 6 categories
"dhe", //health status
"dhm", //mental health status
"dhe_mcs", //mental health - SF12 score MCS
"dhe_pcs", //physical health - SF12 score PCS
"dhe_mcssp", //mental health - SF12 score MCS (partner)
"dhe_pcssp", //physical health - SF12 score PCS (partner)
"dhm", //mental health status
"scghq2_dv", //mental health status case based
"dhm_ghq", //mental health status case based dummy (1 = psychologically distressed)
"dls", //life satisfaction
"dhe_mcs", //mental health - SF12 score MCS
"dhe_pcs", //physical health - SF12 score PCS
"financial_distress", //financial distress
"dcpyy", //years in partnership
"dcpagdf", //partners age difference
Expand All @@ -140,7 +143,7 @@ public class Parameters {
"ypnoab", //gross personal pension (public / occupational) income
"yplgrs_dv", //gross personal employment income
"ynbcpdf_dv", //difference partner income
"dlltsd", //long-term sick or disabled
"dlltsd01", //long-term sick or disabled (we use this -and not dlltsd- in the DataParser)
"sedex", //year left education
"stm", //system variable - year
"swv", //system variable - wave
Expand Down Expand Up @@ -278,7 +281,7 @@ else if(numberOfChildren <= 5) {
//public static int MAX_AGE_IN_EDUCATION;// = MAX_AGE;//30; // Max age a person can stay in education //Cannot set here, as MAX_AGE is not known yet. Now set to MAX_AGE in buildObjects in Model class.
//public static int MAX_AGE_MARRIAGE;// = MAX_AGE;//75; // Max age a person can marry //Cannot set here, as MAX_AGE is not known yet. Now set to MAX_AGE in buildObjects in Model class.
private static int MIN_START_YEAR = 2011; //Minimum allowed starting point. Should correspond to the oldest initial population.
private static int MAX_START_YEAR = 2021; //Maximum allowed starting point. Should correspond to the most recent initial population.
private static int MAX_START_YEAR = 2023; //Maximum allowed starting point. Should correspond to the most recent initial population.
public static int startYear;
public static int endYear;
private static final int MIN_START_YEAR_TESTING = 2019;
Expand All @@ -298,7 +301,7 @@ else if(numberOfChildren <= 5) {

public static final boolean systemOut = true;

//Bootstrap all the regression coefficients if true, or only the female labour participation regressions when false
//Bootstrap all the regression coefficients if true
public static final boolean bootstrapAll = false;

//Scheduling
Expand Down Expand Up @@ -677,8 +680,8 @@ else if(numberOfChildren <= 5) {
/////////////////////////////////////////////////////////////////// REGRESSION OBJECTS //////////////////////////////////////////

//Health
private static OrderedRegression regHealthH1a;
private static OrderedRegression regHealthH1b;
private static GeneralisedOrderedRegression regHealthH1a;
private static GeneralisedOrderedRegression regHealthH1b;
private static BinomialRegression regHealthH2b;

//Social care
Expand Down Expand Up @@ -737,7 +740,7 @@ else if(numberOfChildren <= 5) {
//Education
private static BinomialRegression regEducationE1a;
private static BinomialRegression regEducationE1b;
private static OrderedRegression regEducationE2a;
private static GeneralisedOrderedRegression regEducationE2a;

//Partnership
private static BinomialRegression regPartnershipU1a;
Expand Down Expand Up @@ -1108,8 +1111,8 @@ public static void loadParameters(Country country, int maxAgeModel, boolean enab
columnsIncomeI3b_selection = 22;
columnsLeaveHomeP1a = 19;
columnsHomeownership = 32;
columnsRetirementR1a = 19;
columnsRetirementR1b = 24;
columnsRetirementR1a = 33;
columnsRetirementR1b = 37;
columnsChildcareC1a = 37;
columnsChildcareC1b = 37;
columnsValidationStudentsByAge = 10;
Expand All @@ -1129,14 +1132,14 @@ public static void loadParameters(Country country, int maxAgeModel, boolean enab
columnsValidationLabourSupplyByEducation = 3;
}
else if(country.equals(Country.UK)) {
columnsWagesMalesNE = 30;
columnsWagesMalesE = 31;
columnsWagesFemalesNE = 30;
columnsWagesFemalesE = 31;
columnsEmploymentSelectionMalesNE = 30;
columnsEmploymentSelectionMalesE = 29;
columnsEmploymentSelectionFemalesNE = 30;
columnsEmploymentSelectionFemalesE = 29;
columnsWagesMalesNE = 33;
columnsWagesMalesE = 34;
columnsWagesFemalesNE = 33;
columnsWagesFemalesE = 34;
columnsEmploymentSelectionMalesNE = 32;
columnsEmploymentSelectionMalesE = 32;
columnsEmploymentSelectionFemalesNE = 32;
columnsEmploymentSelectionFemalesE = 32;
columnsLabourSupplyUtilityMales = 19;
columnsLabourSupplyUtilityFemales = 12;
columnsLabourSupplyUtilityMalesWithDependent = 23;
Expand All @@ -1146,9 +1149,9 @@ else if(country.equals(Country.UK)) {
columnsLabourSupplyUtilityCouples = 64;
columnsLabourCovid19_SE = 1;
columnsLabourCovid19_2a_processes = 1;
columnsHealthH1a = 28;
columnsHealthH1b = 35;
columnsHealthH2b = 35;
columnsHealthH1a = 31;
columnsHealthH1b = 110;
columnsHealthH2b = 36;
columnsHealthHM1 = 30;
columnsHealthHM2Males = 15;
columnsHealthHM2Females = 15;
Expand Down Expand Up @@ -1185,31 +1188,31 @@ else if(country.equals(Country.UK)) {
columnsUnemploymentU1c = 19;
columnsUnemploymentU1d = 19;
columnsFinancialDistress = 50;
columnsEducationE1a = 19;
columnsEducationE1b = 25;
columnsEducationE2a = 11;
columnsPartnershipU1a = 27;
columnsPartnershipU1b = 31;
columnsEducationE1a = 24;
columnsEducationE1b = 29;
columnsEducationE2a = 24;
columnsPartnershipU1a = 16;
columnsPartnershipU1b = 29;
columnsPartnershipU2b = 38;
columnsFertilityF1a = 6;
columnsFertilityF1b = 26;
columnsFertilityF1b = 36;
columnsIncomeI1a = 19; //*
columnsIncomeI1b = 31; //*
columnsIncomeI3a = 20;
columnsIncomeI3b = 29;
columnsIncomeI3a = 26;
columnsIncomeI3b = 36;
columnsIncomeI3c = 28; //*
columnsIncomeI4a = 24; //*
columnsIncomeI4b = 25;
columnsIncomeI5a = 25;
columnsIncomeI5b = 25;
columnsIncomeI4b = 31;
columnsIncomeI5a = 30;
columnsIncomeI5b = 31;
columnsIncomeI6a = 22; //*
columnsIncomeI6b = 22; //*
columnsIncomeI3a_selection = 20;
columnsIncomeI3b_selection = 29;
columnsIncomeI3a_selection = 26;
columnsIncomeI3b_selection = 36;
columnsLeaveHomeP1a = 25;
columnsHomeownership = 33;
columnsRetirementR1a = 26;
columnsRetirementR1b = 31;
columnsHomeownership = 41;
columnsRetirementR1a = 33;
columnsRetirementR1b = 37;
columnsChildcareC1a = 37;
columnsChildcareC1b = 37;
columnsValidationStudentsByAge = 10;
Expand Down Expand Up @@ -1408,7 +1411,7 @@ else if (country.equals(Country.IT)) {
}

//Partnership - parameters for matching based on wage and age differential
meanCovarianceParametricMatching = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "scenario_parametricMatching.xlsx", countryString, 1, 1);
meanCovarianceParametricMatching = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "scenario_parametricMatching.xlsx", "Parameters", 1, 1);

//Fertility
if (country.equals(Country.UK)) {
Expand All @@ -1420,11 +1423,11 @@ else if (country.equals(Country.IT)) {
}

//Income
coeffCovarianceIncomeI3a = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3a", 1, columnsIncomeI3a);
coeffCovarianceIncomeI3b = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3b", 1, columnsIncomeI3b);
coeffCovarianceIncomeI4b = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I4b", 1, columnsIncomeI4b);
coeffCovarianceIncomeI5a_selection = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I5a", 1, columnsIncomeI5a);
coeffCovarianceIncomeI5b_amount = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I5b", 1, columnsIncomeI5b);
coeffCovarianceIncomeI3a = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3a_amount", 1, columnsIncomeI3a);
coeffCovarianceIncomeI3b = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3b_amount", 1, columnsIncomeI3b);
coeffCovarianceIncomeI4b = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I4b_amount", 1, columnsIncomeI4b);
coeffCovarianceIncomeI5a_selection = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I5a_selection", 1, columnsIncomeI5a);
coeffCovarianceIncomeI5b_amount = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I5a_amount", 1, columnsIncomeI5b);
coeffCovarianceIncomeI3a_selection = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3a_selection", 1, columnsIncomeI3a_selection);
coeffCovarianceIncomeI3b_selection = ExcelAssistant.loadCoefficientMap(Parameters.getInputDirectory() + "reg_income.xlsx", countryString + "_I3b_selection", 1, columnsIncomeI3b_selection);

Expand Down Expand Up @@ -1556,8 +1559,8 @@ else if (country.equals(Country.IT)) {
}

//Health
regHealthH1a = new OrderedRegression<>(RegressionType.GenOrderedLogit, Dhe.class, coeffCovarianceHealthH1a);
regHealthH1b = new OrderedRegression<>(RegressionType.GenOrderedLogit, Dhe.class, coeffCovarianceHealthH1b);
regHealthH1a = new GeneralisedOrderedRegression<>(RegressionType.GenOrderedLogit, Dhe.class, coeffCovarianceHealthH1a);
regHealthH1b = new GeneralisedOrderedRegression<>(RegressionType.GenOrderedLogit, Dhe.class, coeffCovarianceHealthH1b);
regHealthH2b = new BinomialRegression(RegressionType.Probit, Indicator.class, coeffCovarianceHealthH2b);

//Social care
Expand Down Expand Up @@ -1611,7 +1614,7 @@ else if (country.equals(Country.IT)) {

regEducationE1a = new BinomialRegression(RegressionType.Probit, Indicator.class, coeffCovarianceEducationE1a);
regEducationE1b = new BinomialRegression(RegressionType.Probit, Indicator.class, coeffCovarianceEducationE1b);
regEducationE2a = new OrderedRegression<>(RegressionType.GenOrderedLogit, Education.class, coeffCovarianceEducationE2a);
regEducationE2a = new GeneralisedOrderedRegression<>(RegressionType.GenOrderedLogit, Education.class, coeffCovarianceEducationE2a);

//Partnership
if (country.equals(Country.UK)) {
Expand Down Expand Up @@ -2074,8 +2077,8 @@ public static void setEmploymentsFurloughedFlex(MultiKeyCoefficientMap employmen
Parameters.employmentsFurloughedFlex = employmentsFurloughedFlex;
}

public static OrderedRegression getRegHealthH1a() { return regHealthH1a; }
public static OrderedRegression getRegHealthH1b() { return regHealthH1b; }
public static GeneralisedOrderedRegression getRegHealthH1a() { return regHealthH1a; }
public static GeneralisedOrderedRegression getRegHealthH1b() { return regHealthH1b; }
public static BinomialRegression getRegHealthH2b() { return regHealthH2b; }

public static BinomialRegression getRegReceiveCareS1a() { return regReceiveCareS1a; }
Expand Down Expand Up @@ -2125,7 +2128,7 @@ public static void setEmploymentsFurloughedFlex(MultiKeyCoefficientMap employmen

public static BinomialRegression getRegEducationE1a() {return regEducationE1a;}
public static BinomialRegression getRegEducationE1b() {return regEducationE1b;}
public static OrderedRegression getRegEducationE2a() {return regEducationE2a;}
public static GeneralisedOrderedRegression getRegEducationE2a() {return regEducationE2a;}

public static LinearRegression getRegEQ5D() { return regHealthEQ5D; };

Expand Down
6 changes: 3 additions & 3 deletions src/main/java/simpaths/data/RegressionName.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public enum RegressionName {

EducationE1a(RegressionType.Probit),
EducationE1b(RegressionType.Probit),
EducationE2a(RegressionType.OrderedLogit),
EducationE2a(RegressionType.GenOrderedLogit),

FertilityF1a(RegressionType.Probit),
FertilityF1b(RegressionType.Probit),
Expand All @@ -18,8 +18,8 @@ public enum RegressionName {
PartnershipU1b(RegressionType.Probit),
PartnershipU2b(RegressionType.Probit),

HealthH1a(RegressionType.OrderedLogit),
HealthH1b(RegressionType.OrderedLogit),
HealthH1a(RegressionType.GenOrderedLogit),
HealthH1b(RegressionType.GenOrderedLogit),
HealthH2b(RegressionType.Probit),

HealthHM1Level(RegressionType.Linear),
Expand Down
23 changes: 12 additions & 11 deletions src/main/java/simpaths/data/startingpop/DataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,10 +162,10 @@ private static void parse(String inputFileLocation, String inputFileName, Connec

//DEMOGRAPHIC: Long-term sick or disabled (to be used with Indicator enum when defined in Person class)
+ "ALTER TABLE " + personTable + " ADD sick_longterm VARCHAR_IGNORECASE;"
+ "UPDATE " + personTable + " SET sick_longterm = 'False' WHERE dlltsd = 0;"
+ "UPDATE " + personTable + " SET sick_longterm = 'True' WHERE dlltsd = 1;"
+ "ALTER TABLE " + personTable + " DROP COLUMN dlltsd;"
+ "ALTER TABLE " + personTable + " ALTER COLUMN sick_longterm RENAME TO dlltsd;"
+ "UPDATE " + personTable + " SET sick_longterm = 'False' WHERE dlltsd01 = 0;"
+ "UPDATE " + personTable + " SET sick_longterm = 'True' WHERE dlltsd01 = 1;"
+ "ALTER TABLE " + personTable + " DROP COLUMN dlltsd01;"
+ "ALTER TABLE " + personTable + " ALTER COLUMN sick_longterm RENAME TO dlltsd;" // There are two versions of this variable in the Initial Population files. We use the dlltsd01, but we rename it dlltsd fo simplicity

//DEMOGRAPHIC: Need social care (to be used with Indicator enum when defined in Person class)
+ "ALTER TABLE " + personTable + " ADD need_care VARCHAR_IGNORECASE;"
Expand All @@ -176,13 +176,14 @@ private static void parse(String inputFileLocation, String inputFileName, Connec

//DEMOGRAPHIC: Ethnicity
+ "ALTER TABLE " + personTable + " ADD ethnicity VARCHAR_IGNORECASE;"
+ "UPDATE " + personTable + " SET ethnicity = 'White' WHERE dot = 1;"
+ "UPDATE " + personTable + " SET ethnicity = 'Mixed' WHERE dot = 2;"
+ "UPDATE " + personTable + " SET ethnicity = 'Asian' WHERE dot = 3;"
+ "UPDATE " + personTable + " SET ethnicity = 'Black' WHERE dot = 4;"
+ "UPDATE " + personTable + " SET ethnicity = 'Other' WHERE dot = 5;"
+ "ALTER TABLE " + personTable + " DROP COLUMN dot;"
+ "ALTER TABLE " + personTable + " ALTER COLUMN ethnicity RENAME TO dot;"
+ "UPDATE " + personTable + " SET ethnicity = 'White' WHERE dot01 = 1;"
+ "UPDATE " + personTable + " SET ethnicity = 'Mixed' WHERE dot01 = 2;"
+ "UPDATE " + personTable + " SET ethnicity = 'Asian' WHERE dot01 = 3;"
+ "UPDATE " + personTable + " SET ethnicity = 'Black' WHERE dot01 = 4;"
+ "UPDATE " + personTable + " SET ethnicity = 'Other' WHERE dot01 = 5;"
+ "UPDATE " + personTable + " SET ethnicity = 'Missing' WHERE dot01 = 6;"
+ "ALTER TABLE " + personTable + " DROP COLUMN dot01;"
+ "ALTER TABLE " + personTable + " ALTER COLUMN ethnicity RENAME TO dot01;"

//SYSTEM: Year left education (to be used with Indicator enum when defined in Person class)
+ "ALTER TABLE " + personTable + " ADD education_left VARCHAR_IGNORECASE;"
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/simpaths/experiment/SimPathsCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,13 @@ public int compare(Pair<BenefitUnit, Double> pair1, Pair<BenefitUnit, Double> pa
}
}

if (median == null) {
throw new IllegalStateException(
"Median could not be calculated — totalWeight=" + totalWeight +
", households considered=" + arrHouse_eqHouseholdDispIncome.size()
);
}

double atRiskOfPovertyThreshold = median * 0.6;
// log.info("atRiskOfPovertyThreshold = " + atRiskOfPovertyThreshold);
stats.setMedianEquivalisedHouseholdDisposableIncome(median); //Save median household equivalised disposable income in statistics object
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/simpaths/experiment/SimPathsMultiRun.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,10 @@ public static void main(String[] args) {
try {
if (lastDatabaseCountryAndYear.keySet().stream().anyMatch(key -> key.toString().equals("MultiKey[IT]"))) {
countryString = "Italy";
country = Country.IT;
} else {
countryString = "United Kingdom";
country = Country.UK;
}
String valueYear = lastDatabaseCountryAndYear.getValue(country.toString()).toString();
startYear = Integer.parseInt(valueYear);
Expand Down
Loading
Loading