From 96f9784aceeeb84728200d7f0e2987f507ef8bc6 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 16:36:01 +0000 Subject: [PATCH 1/8] re-add indexing for timing check --- src/main/java/simpaths/data/startingpop/DataParser.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/simpaths/data/startingpop/DataParser.java b/src/main/java/simpaths/data/startingpop/DataParser.java index 4beaa3da0..c1ccdc94d 100644 --- a/src/main/java/simpaths/data/startingpop/DataParser.java +++ b/src/main/java/simpaths/data/startingpop/DataParser.java @@ -269,7 +269,11 @@ private static void parse(String inputFileLocation, String inputFileName, Connec + "ALTER TABLE " + personTable + " ADD COLUMN prid INT DEFAULT 0;" + "ALTER TABLE " + personTable + " ALTER COLUMN idhh RENAME TO idhousehold;" - //Re-order by id + + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_bukey ON " + personTable + " (buid, butime, burun, prid);" + + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_idhousehold ON " + personTable + " (idhousehold);" + + + //Re-order by id + "SELECT * FROM " + personTable + " ORDER BY id;" ); From 42a97743e20e1122cb174e3acd0293279bc4debb Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 17:12:57 +0000 Subject: [PATCH 2/8] Revert "re-add indexing for timing check" This reverts commit 96f9784aceeeb84728200d7f0e2987f507ef8bc6. --- src/main/java/simpaths/data/startingpop/DataParser.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/simpaths/data/startingpop/DataParser.java b/src/main/java/simpaths/data/startingpop/DataParser.java index c1ccdc94d..4beaa3da0 100644 --- a/src/main/java/simpaths/data/startingpop/DataParser.java +++ b/src/main/java/simpaths/data/startingpop/DataParser.java @@ -269,11 +269,7 @@ private static void parse(String inputFileLocation, String inputFileName, Connec + "ALTER TABLE " + personTable + " ADD COLUMN prid INT DEFAULT 0;" + "ALTER TABLE " + personTable + " ALTER COLUMN idhh RENAME TO idhousehold;" - + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_bukey ON " + personTable + " (buid, butime, burun, prid);" - + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_idhousehold ON " + personTable + " (idhousehold);" - - - //Re-order by id + //Re-order by id + "SELECT * FROM " + personTable + " ORDER BY id;" ); From 074a285c020d0d529cda245fee535232c43d7bf3 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 21:47:37 +0000 Subject: [PATCH 3/8] speed up #1 on `populateTaxdbReferences` Removed 'LEFT JOIN' statement to leave to lazy loading --- src/main/java/simpaths/model/SimPathsModel.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index 09e0913b9..32928bc75 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3227,12 +3227,12 @@ private static void populateTaxdbReferences() { EntityManager em = Persistence.createEntityManagerFactory("tax-database", propertyMap).createEntityManager(); txn = em.getTransaction(); txn.begin(); - String query = "SELECT DISTINCT tu FROM DonorTaxUnit tu LEFT JOIN FETCH tu.policies tp ORDER BY tp.originalIncomePerMonth"; - //String query = "SELECT tu FROM DonorTaxUnit tu"; +// String query = "SELECT DISTINCT tu FROM DonorTaxUnit tu LEFT JOIN FETCH tu.policies tp ORDER BY tp.originalIncomePerMonth"; + String query = "SELECT tu FROM DonorTaxUnit tu"; List donorPool = em.createQuery(query).getResultList(); -// donorPool.sort(Comparator.comparingDouble(tu -> -// tu.getPolicyBySystemYear(Parameters.BASE_PRICE_YEAR).getOriginalIncomePerMonth() -// )); + donorPool.sort(Comparator.comparingDouble(tu -> + tu.getPolicyBySystemYear(Parameters.BASE_PRICE_YEAR).getOriginalIncomePerMonth() + )); System.out.println("Completed accessing donor data from the database"); From d146dbf8b3f6a05ab146f70d63a26aa2fb720547 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 22:05:11 +0000 Subject: [PATCH 4/8] speed up #2 on `loadStartingPopulation` Removed 'LEFT JOIN' --- src/main/java/simpaths/model/SimPathsModel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index 32928bc75..f5bb32917 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3446,8 +3446,8 @@ private List loadStartingPopulation() { EntityManager em = Persistence.createEntityManagerFactory("starting-population", propertyMap).createEntityManager(); txn = em.getTransaction(); txn.begin(); -// String query = "SELECT households FROM Household households"; - String query = "SELECT DISTINCT households FROM Household households LEFT JOIN FETCH households.benefitUnits benefitUnits LEFT JOIN FETCH benefitUnits.members members"; + String query = "SELECT households FROM Household households"; +// String query = "SELECT DISTINCT households FROM Household households LEFT JOIN FETCH households.benefitUnits benefitUnits LEFT JOIN FETCH benefitUnits.members members"; log.info("Submitting SQL query: " + query); households = em.createQuery(query).getResultList(); log.info("Query complete"); From 64e3e329c5b982cdb915281fc51d250209a87879 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 22:14:48 +0000 Subject: [PATCH 5/8] speed up #3 on `getProcessed` Removed 'LEFT JOIN' and fetched only households only matching only after confirming db is right --- .../java/simpaths/model/SimPathsModel.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index f5bb32917..b5ccbbc01 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3387,7 +3387,7 @@ private Processed getProcessed() { private Processed getProcessed(Country country, int startYear, int popSize, boolean ignoreTargetsAtPopulationLoad) { Processed processed = null; - //Processed processed_return = null; + Processed processed_return = null; EntityTransaction txn = null; try { @@ -3398,8 +3398,8 @@ private Processed getProcessed(Country country, int startYear, int popSize, bool EntityManager em = Persistence.createEntityManagerFactory("starting-population", propertyMap).createEntityManager(); txn = em.getTransaction(); txn.begin(); - String query = "SELECT DISTINCT processed FROM Processed processed LEFT JOIN FETCH processed.households households LEFT JOIN FETCH households.benefitUnits benefitUnits LEFT JOIN FETCH benefitUnits.members members WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad + " ORDER BY households.key.id"; -// String query = "SELECT processed FROM Processed processed WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad; +// String query = "SELECT DISTINCT processed FROM Processed processed LEFT JOIN FETCH processed.households households LEFT JOIN FETCH households.benefitUnits benefitUnits LEFT JOIN FETCH benefitUnits.members members WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad + " ORDER BY households.key.id"; + String query = "SELECT processed FROM Processed processed WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad; log.info("Submitting SQL query: " + query); List processedList = em.createQuery(query).getResultList(); @@ -3412,13 +3412,13 @@ private Processed getProcessed(Country country, int startYear, int popSize, bool processed.resetDependents(); // // Now fetch households for THIS specific Processed instance only -// processed_return = em.createQuery( -// "SELECT p FROM Processed p LEFT JOIN FETCH p.households h WHERE p = :proc ORDER BY h.key.id", -// Processed.class) -// .setParameter("proc", processed) -// .getSingleResult(); -// -// processed_return.resetDependents(); + processed_return = em.createQuery( + "SELECT p FROM Processed p LEFT JOIN FETCH p.households h WHERE p = :proc ORDER BY h.key.id", + Processed.class) + .setParameter("proc", processed) + .getSingleResult(); + + processed_return.resetDependents(); } // close database connection From b93ee48851eaa9cdd3c146d3d87aebbb4b5c2670 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 22:20:02 +0000 Subject: [PATCH 6/8] speed up #4 on `getLifetimeIncomes` Remove 'LEFT JOIN' and make all fetches LAZY for birthcohort, individuals and incomes --- src/main/java/simpaths/model/SimPathsModel.java | 3 ++- .../java/simpaths/model/lifetime_incomes/AnnualIncome.java | 2 +- .../java/simpaths/model/lifetime_incomes/BirthCohort.java | 2 +- .../java/simpaths/model/lifetime_incomes/Individual.java | 6 +++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index b5ccbbc01..46f2c555d 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3359,7 +3359,8 @@ private LifetimeIncomeImputation getLifetimeIncomes(int year) { EntityManager em = Persistence.createEntityManagerFactory("lifetime-incomes", propertyMap).createEntityManager(); txn = em.getTransaction(); txn.begin(); - String query = "SELECT DISTINCT cohort FROM BirthCohort cohort LEFT JOIN FETCH cohort.individuals individuals LEFT JOIN FETCH individuals.incomes incomes"; + String query = "SELECT DISTINCT cohort FROM BirthCohort cohort"; +// String query = "SELECT DISTINCT cohort FROM BirthCohort cohort LEFT JOIN FETCH cohort.individuals individuals LEFT JOIN FETCH individuals.incomes incomes"; System.out.println("Submitting SQL query"); log.info("Submitting SQL query: " + query); List cohorts = em.createQuery(query).getResultList(); diff --git a/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java b/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java index 17aab57bc..4e5c099c7 100644 --- a/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java +++ b/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java @@ -10,7 +10,7 @@ public class AnnualIncome implements IDoubleSource { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @ManyToOne(fetch = FetchType.EAGER, cascade=CascadeType.REFRESH) + @ManyToOne(fetch = FetchType.LAZY, cascade=CascadeType.REFRESH) @JoinColumns({ @JoinColumn(name = "individual_id", referencedColumnName = "id") }) diff --git a/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java b/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java index fc27de92f..e6e4236e9 100644 --- a/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java +++ b/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java @@ -14,7 +14,7 @@ public class BirthCohort { * ATTRIBUTES */ @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @OneToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL, mappedBy = "cohort") + @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "cohort") @OrderBy("id ASC") private Set individuals = new LinkedHashSet<>(); diff --git a/src/main/java/simpaths/model/lifetime_incomes/Individual.java b/src/main/java/simpaths/model/lifetime_incomes/Individual.java index e92b98384..29e34a036 100644 --- a/src/main/java/simpaths/model/lifetime_incomes/Individual.java +++ b/src/main/java/simpaths/model/lifetime_incomes/Individual.java @@ -20,15 +20,15 @@ public class Individual implements IDoubleSource { * ATTRIBUTES */ @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @ManyToOne(fetch = FetchType.EAGER, cascade=CascadeType.REFRESH) + @ManyToOne(fetch = FetchType.LAZY, cascade=CascadeType.REFRESH) @JoinColumns({ @JoinColumn(name = "cohort_id", referencedColumnName = "id") }) private BirthCohort cohort; - @OneToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL, mappedBy = "individual") + @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "individual") @OrderBy("id ASC") private Set incomes = new LinkedHashSet<>(); - @OneToMany(fetch = FetchType.EAGER, mappedBy = "ltIncomeDonor") + @OneToMany(fetch = FetchType.LAZY, mappedBy = "ltIncomeDonor") private Set persons = new LinkedHashSet<>(); @Transient int year; From c3bfd6b743a60d41037df63a37467ce3d28198ec Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 22:33:28 +0000 Subject: [PATCH 7/8] Revert "speed up #1 on `populateTaxdbReferences`" This reverts commit 074a285c020d0d529cda245fee535232c43d7bf3. --- src/main/java/simpaths/model/SimPathsModel.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index 46f2c555d..83d963acf 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3227,12 +3227,12 @@ private static void populateTaxdbReferences() { EntityManager em = Persistence.createEntityManagerFactory("tax-database", propertyMap).createEntityManager(); txn = em.getTransaction(); txn.begin(); -// String query = "SELECT DISTINCT tu FROM DonorTaxUnit tu LEFT JOIN FETCH tu.policies tp ORDER BY tp.originalIncomePerMonth"; - String query = "SELECT tu FROM DonorTaxUnit tu"; + String query = "SELECT DISTINCT tu FROM DonorTaxUnit tu LEFT JOIN FETCH tu.policies tp ORDER BY tp.originalIncomePerMonth"; + //String query = "SELECT tu FROM DonorTaxUnit tu"; List donorPool = em.createQuery(query).getResultList(); - donorPool.sort(Comparator.comparingDouble(tu -> - tu.getPolicyBySystemYear(Parameters.BASE_PRICE_YEAR).getOriginalIncomePerMonth() - )); +// donorPool.sort(Comparator.comparingDouble(tu -> +// tu.getPolicyBySystemYear(Parameters.BASE_PRICE_YEAR).getOriginalIncomePerMonth() +// )); System.out.println("Completed accessing donor data from the database"); From 618658b545dddfaf1024617c2c6599d8fa9c6694 Mon Sep 17 00:00:00 2001 From: Andy Baxter Date: Tue, 11 Nov 2025 16:36:01 +0000 Subject: [PATCH 8/8] re-add indexing for timing check --- src/main/java/simpaths/data/startingpop/DataParser.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/simpaths/data/startingpop/DataParser.java b/src/main/java/simpaths/data/startingpop/DataParser.java index 4beaa3da0..c1ccdc94d 100644 --- a/src/main/java/simpaths/data/startingpop/DataParser.java +++ b/src/main/java/simpaths/data/startingpop/DataParser.java @@ -269,7 +269,11 @@ private static void parse(String inputFileLocation, String inputFileName, Connec + "ALTER TABLE " + personTable + " ADD COLUMN prid INT DEFAULT 0;" + "ALTER TABLE " + personTable + " ALTER COLUMN idhh RENAME TO idhousehold;" - //Re-order by id + + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_bukey ON " + personTable + " (buid, butime, burun, prid);" + + "CREATE INDEX IF NOT EXISTS idx_" + personTable + "_idhousehold ON " + personTable + " (idhousehold);" + + + //Re-order by id + "SELECT * FROM " + personTable + " ORDER BY id;" );