From 3fd213e0b98a9eea058b99e4fffc2de8ea61cd5e Mon Sep 17 00:00:00 2001 From: suhlrich <43877159+suhlrich@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:49:59 -0600 Subject: [PATCH] reduce cpu load on dequeue (hopefully). it still searches for saved_local trials over the window for processing. I'm going to drop that to 4 days - i don't think it ever takes that long to get out of the queue the first time. --- .../migrations/0048_video_saved_local_index.py | 18 ++++++++++++++++++ mcserver/models.py | 2 +- mcserver/views.py | 17 ++++++++++------- 3 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 mcserver/migrations/0048_video_saved_local_index.py diff --git a/mcserver/migrations/0048_video_saved_local_index.py b/mcserver/migrations/0048_video_saved_local_index.py new file mode 100644 index 0000000..6cb1981 --- /dev/null +++ b/mcserver/migrations/0048_video_saved_local_index.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.14 on 2026-06-30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("mcserver", "0047_video_saved_local"), + ] + + operations = [ + migrations.AlterField( + model_name="video", + name="saved_local", + field=models.BooleanField(db_index=True, default=False), + ), + ] diff --git a/mcserver/models.py b/mcserver/models.py index bd8a1ed..8264754 100644 --- a/mcserver/models.py +++ b/mcserver/models.py @@ -180,7 +180,7 @@ class Video(models.Model): keypoints = models.FileField(blank=True, null=True) parameters = models.JSONField(blank=True, null=True) isLidar = models.BooleanField(default=False) - saved_local = models.BooleanField(default=False) + saved_local = models.BooleanField(default=False, db_index=True) created_at = models.DateTimeField(auto_now_add=True, db_index=True) updated_at = models.DateTimeField(auto_now=True, db_index=True) diff --git a/mcserver/views.py b/mcserver/views.py index 8042e10..cacadcd 100644 --- a/mcserver/views.py +++ b/mcserver/views.py @@ -1665,23 +1665,26 @@ def dequeue(self, request): - # Trials are considered waiting-for-upload when at least one video is missing. - # Keep the 15-minute grace period for normal uploads, but always wait when - # a missing video is explicitly marked as saved locally on phones for upload later. - missing_video_q = Q(video='') | Q(video__isnull=True) + # Trials are waiting-for-upload when a missing video was updated + # recently. Do not dequeue trials with non-uploaded videos + # or saved-local videos, within the 7 day updated-at window. + active_trial_cutoff = timezone.now() + timedelta(days=-4) + recent_video_cutoff = timezone.now() + timedelta(minutes=-15) + missing_video_q = Q(video='') not_uploaded = Video.objects.filter( missing_video_q, + trial__updated_at__gte=active_trial_cutoff, ).filter( - Q(updated_at__gte=timezone.now() + timedelta(minutes=-15)) | + Q(updated_at__gte=recent_video_cutoff) | Q(saved_local=True) ).values_list("trial__id", flat=True) if isMonoQuery == 'False': - uploaded_trials = Trial.objects.filter(updated_at__gte=timezone.now() + timedelta(days=-7)).exclude( + uploaded_trials = Trial.objects.filter(updated_at__gte=active_trial_cutoff).exclude( id__in=not_uploaded).exclude(session__isMono=True) else: - uploaded_trials = Trial.objects.filter(updated_at__gte=timezone.now() + timedelta(days=-7)).exclude( + uploaded_trials = Trial.objects.filter(updated_at__gte=active_trial_cutoff).exclude( id__in=not_uploaded).filter(session__isMono=True) if workerType != 'dynamic':