Skip to content

261 slow multirun performance after database setup with simpathstart #267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dav-sonn merged 3 commits into from
Oct 13, 2025
Merged

261 slow multirun performance after database setup with simpathstart #267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
464ff34
experimental - loading processed lazily #261
andrewbaxter439 Oct 9, 2025
16af2f5
Fixed getProcessed SQL call to return correct households #261
andrewbaxter439 Oct 10, 2025
46ed923
lazy fetching households #261
andrewbaxter439 Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/main/java/simpaths/data/startingpop/Processed.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@


import jakarta.persistence.*;
import org.hibernate.annotations.Fetch;
import org.hibernate.annotations.FetchMode;
import simpaths.model.BenefitUnit;
import simpaths.model.Household;
import simpaths.model.Person;
Expand All @@ -22,8 +24,9 @@ public class Processed {
* ATTRIBUTES
*/
@Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id;
@OneToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL, mappedBy = "processed")
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "processed")
@OrderBy("key ASC")
@Fetch(FetchMode.SUBSELECT)
private Set<Household> households = new LinkedHashSet<>();

@Enumerated(EnumType.STRING) Country country;
Expand Down
13 changes: 11 additions & 2 deletions src/main/java/simpaths/model/SimPathsModel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3314,6 +3314,7 @@ private Processed getProcessed() {
private Processed getProcessed(Country country, int startYear, int popSize, boolean ignoreTargetsAtPopulationLoad) {

Processed processed = null;
Processed processed_return = null;

EntityTransaction txn = null;
try {
Expand All @@ -3324,7 +3325,7 @@ private Processed getProcessed(Country country, int startYear, int popSize, bool
EntityManager em = Persistence.createEntityManagerFactory("starting-population", propertyMap).createEntityManager();
txn = em.getTransaction();
txn.begin();
String query = "SELECT processed FROM Processed processed LEFT JOIN FETCH processed.households households LEFT JOIN FETCH households.benefitUnits benefitUnits LEFT JOIN FETCH benefitUnits.members members WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad + " ORDER BY households.key.id";
String query = "SELECT processed FROM Processed processed WHERE processed.startYear = " + startYear + " AND processed.popSize = " + popSize + " AND processed.country = " + country + " AND processed.noTargets = " + ignoreTargetsAtPopulationLoad;
log.info("Submitting SQL query: " + query);

List<Processed> processedList = em.createQuery(query).getResultList();
Expand All @@ -3334,7 +3335,15 @@ private Processed getProcessed(Country country, int startYear, int popSize, bool
if (processedList.size()>1)
throw new RuntimeException("more than one relevant dataset returned from database");
processed = processedList.get(0);
processed.resetDependents();

// Now fetch households for THIS specific Processed instance only
processed_return = em.createQuery(
"SELECT p FROM Processed p LEFT JOIN FETCH p.households h WHERE p = :proc ORDER BY h.key.id",
Processed.class)
.setParameter("proc", processed)
.getSingleResult();

processed_return.resetDependents();
}

// close database connection
Expand Down