From 3dce696a53b156c05ebd750c5a73404f09be0788 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 13:15:40 +0100 Subject: [PATCH 1/7] multirun `-DBsetup` option now checks whether training data is used --- src/main/java/simpaths/data/Parameters.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 9815aa3a9..f9f444803 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -8,6 +8,7 @@ import java.util.*; // import plug-in packages +import org.apache.commons.io.FileUtils; import simpaths.data.startingpop.DataParser; import simpaths.model.AnnuityRates; import simpaths.model.enums.*; @@ -3318,6 +3319,11 @@ public static void databaseSetup(Country country, boolean executeWithGui, int st String filePath = "./input" + File.separator + "input.mv.db"; safeDelete(filePath); + // Detect if data available; set to testing data if not + Collection testList = FileUtils.listFiles(new File(Parameters.getInputDirectoryInitialPopulations()), new String[]{"csv"}, false); + if (testList.size()==0) + Parameters.setTrainingFlag(true); + // populate new database for starting data DataParser.databaseFromCSV(country, executeWithGui); // Initial database tables From a982ac36bcd69ef9aeb189ccee19adedb3d9620d Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 14:17:55 +0100 Subject: [PATCH 2/7] create test config files --- config/test_create_database.yml | 10 ++++++++++ config/test_run.yml | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 config/test_create_database.yml create mode 100644 config/test_run.yml diff --git a/config/test_create_database.yml b/config/test_create_database.yml new file mode 100644 index 000000000..882e1c9ce --- /dev/null +++ b/config/test_create_database.yml @@ -0,0 +1,10 @@ +# FOR TESTING +# CONFIG FILE TO CREATE NEW DATABASE OBJECTS + +# Arguments of the SimPathsMultiRun object overridden by the command-line +countryString: "United Kingdom" +executeWithGui: false +randomSeed: 606 +startYear: 2019 +endYear: 2030 +popSize: 40000 diff --git a/config/test_run.yml b/config/test_run.yml new file mode 100644 index 000000000..9cecb034e --- /dev/null +++ b/config/test_run.yml @@ -0,0 +1,17 @@ +# This file can be used to override defaults for multirun arguments. +# Arguments of the SimPathsMultiRun object overridden by the command-line + +maxNumberOfRuns: 2 +executeWithGui: false +randomSeed: 100 +startYear: 2019 +endYear: 2022 +popSize: 20000 + +collector_args: + persistStatistics: true + persistStatistics2: true + persistStatistics3: true + persistPersons: false + persistBenefitUnits: false + persistHouseholds: false \ No newline at end of file From 0cff4c26ad4c7c9e5afdf55c1a9dcd325f438ec7 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 14:18:28 +0100 Subject: [PATCH 3/7] change integration tests to setup using multirun db setup --- .../RunSimPathsIntegrationTest.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/test/java/simpaths/integrationtest/RunSimPathsIntegrationTest.java b/src/test/java/simpaths/integrationtest/RunSimPathsIntegrationTest.java index 77b88102c..2216ac68c 100644 --- a/src/test/java/simpaths/integrationtest/RunSimPathsIntegrationTest.java +++ b/src/test/java/simpaths/integrationtest/RunSimPathsIntegrationTest.java @@ -25,12 +25,7 @@ public class RunSimPathsIntegrationTest { @Order(1) void testRunSetup() { runCommand( - "java", "-jar", "singlerun.jar", - "-c", "UK", - "-s", "2017", - "-Setup", - "-g", "false", - "--rewrite-policy-schedule" + "java", "-jar", "multirun.jar", "-DBSetup", "-config", "test_create_database.yml" ); } @@ -48,13 +43,7 @@ void testVerifySetupOutput() { @Order(3) void testRunSimulation() { runCommand( - "java", "-jar", "multirun.jar", - "-p", "20000", - "-s", "2019", - "-e", "2022", - "-r", "100", - "-n", "2", - "-g", "false" + "java", "-jar", "multirun.jar", "-config", "test_run.yml", "-P", "root" ); } From 6ca929cbf519e98e5e36db13c7476ec144d23476 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 14:18:52 +0100 Subject: [PATCH 4/7] changes actions tests to setup via multirun dbsetup --- .github/workflows/SimPathsBuild.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/SimPathsBuild.yml b/.github/workflows/SimPathsBuild.yml index 1beb3c9ad..0cc95bdb1 100644 --- a/.github/workflows/SimPathsBuild.yml +++ b/.github/workflows/SimPathsBuild.yml @@ -58,14 +58,14 @@ jobs: name: simpaths_jars path: . - name: Setup run - run: java -jar singlerun.jar -c UK -s 2017 -Setup -g false --rewrite-policy-schedule + run: java -jar multirun.jar -DBSetup -config test_create_database.yml - name: Check input db exists id: check_file uses: thebinaryfelix/check-file-existence-action@1.0.0 with: files: 'input/input.mv.db, input/EUROMODpolicySchedule.xlsx, input/DatabaseCountryYear.xlsx' - name: Do two runs with persistence to root database - run: java -jar multirun.jar -p 20000 -s 2019 -e 2022 -r 100 -n 2 -g false --persist=root + run: java -jar multirun.jar -config test_run.yml --persist=root run-simpaths-no-persist: needs: build @@ -82,11 +82,11 @@ jobs: name: simpaths_jars path: . - name: Setup run - run: java -jar singlerun.jar -c UK -s 2017 -Setup -g false --rewrite-policy-schedule + run: java -jar multirun.jar -DBSetup -config test_create_database.yml - name: Check input db exists id: check_file uses: thebinaryfelix/check-file-existence-action@1.0.0 with: files: 'input/input.mv.db, input/EUROMODpolicySchedule.xlsx, input/DatabaseCountryYear.xlsx' - name: Do two runs with no persistence - run: java -jar multirun.jar -p 20000 -s 2019 -e 2022 -r 100 -n 2 -g false --persist=none + run: java -jar multirun.jar -config test_run.yml --persist=none From 85251c8019358abb2e987d5d09271e93a5006c4d Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 14:48:03 +0100 Subject: [PATCH 5/7] read training flag in DBSetup to check where policy schedule should be --- src/main/java/simpaths/data/Parameters.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index f9f444803..cbe54a51b 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -1908,8 +1908,14 @@ private static void calculatePopulationGrowthRatiosFromProjections() { public static TreeMap calculateEUROMODpolicySchedule(Country country) { //Load current values for policy description and initiation year - MultiKeyCoefficientMap currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); - TreeMap newEUROMODpolicySchedule = new TreeMap<>(); + MultiKeyCoefficientMap currentEUROMODpolicySchedule; + + if (trainingFlag) { + currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + "EUOROMODoutput" + File.separator + "training" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); + } else { + currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); + } + TreeMap newEUROMODpolicySchedule = new TreeMap<>(); for(Object o: currentEUROMODpolicySchedule.keySet()) { MultiKey k = (MultiKey)o; From 82a9fc2fd2d204c4860610f483dbe70819f31888 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 14:58:39 +0100 Subject: [PATCH 6/7] fix spelling of EUROMOD in training filename finder --- src/main/java/simpaths/data/Parameters.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index cbe54a51b..2019975e2 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -1911,7 +1911,7 @@ public static TreeMap calculateEUROMODpolicySchedule(Country co MultiKeyCoefficientMap currentEUROMODpolicySchedule; if (trainingFlag) { - currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + "EUOROMODoutput" + File.separator + "training" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); + currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + "EUROMODoutput" + File.separator + "training" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); } else { currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); } From f7f7bbfe2ae38158c9bfdd438f4db24cc6a939e0 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Fri, 30 May 2025 16:10:37 +0100 Subject: [PATCH 7/7] change dbsetup to copy training policy schedule if training --- src/main/java/simpaths/data/Parameters.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 2019975e2..e4aa7f04b 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -1911,11 +1911,17 @@ public static TreeMap calculateEUROMODpolicySchedule(Country co MultiKeyCoefficientMap currentEUROMODpolicySchedule; if (trainingFlag) { - currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + "EUROMODoutput" + File.separator + "training" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); - } else { - currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); + File trainingSchedule = new File("input" + File.separator + "EUROMODoutput" + File.separator + "training" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx"); + File runSchedule = new File("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx"); + try { + FileUtils.copyFile(trainingSchedule, runSchedule); + } catch (IOException e) { + System.err.println("Could not replace EUROMODoutput.xlsx from training data"); + } } - TreeMap newEUROMODpolicySchedule = new TreeMap<>(); + + currentEUROMODpolicySchedule = ExcelAssistant.loadCoefficientMap("input" + File.separator + EUROMODpolicyScheduleFilename + ".xlsx", country.toString(), 1, 3); + TreeMap newEUROMODpolicySchedule = new TreeMap<>(); for(Object o: currentEUROMODpolicySchedule.keySet()) { MultiKey k = (MultiKey)o;