From 6331a17c2ef2a55c41222b639f55c7d0f35308ef Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Fri, 28 Mar 2025 17:11:06 +0100 Subject: [PATCH 01/14] [feature] add memory usage to Heartbeat output --- .../management/commands/task_worker.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 5ffcfc8..27956bb 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -1,5 +1,6 @@ import asyncio +import psutil from django.core.management.base import BaseCommand from django_simple_queue.models import Task from django.utils import timezone @@ -63,6 +64,15 @@ def process_task(task_id): print(f"Finished task id: {task_id}") + + + +def log_memory_usage(): + """Returns the memory usage of the current process in MB.""" + process = psutil.Process() + mem_info = process.memory_info() + return round(mem_info.rss / (1024 * 1024), 2) + class Command(BaseCommand): help = 'Executes the enqueued tasks.' @@ -71,7 +81,7 @@ def handle(self, *args, **options): sleep_interval = random.randint(3, 9) while True: time.sleep(sleep_interval) - print(f"{timezone.now()}: Heartbeat..") + print(f"{timezone.now()}: [RAM Usage: {log_memory_usage()} MB] Heartbeat..") queued_task = Task.objects.filter(status=Task.QUEUED).order_by('modified').first() if queued_task: process_task(task_id=queued_task.id) From b2facc21480cbc403805340591f0cb8e10859b70 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Sun, 6 Apr 2025 21:52:32 +0200 Subject: [PATCH 02/14] [fix] run tasks in a subprocess to ensure cleanup Note: the parent's DB connections are closed before forking to avoid corrupting the db connections if the child process writes to them. --- .../management/commands/task_worker.py | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 27956bb..f134ff3 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -1,16 +1,18 @@ import asyncio - -import psutil -from django.core.management.base import BaseCommand -from django_simple_queue.models import Task -from django.utils import timezone import importlib -import time -import json import inspect +import json import random +import time import traceback +from multiprocessing import Process + +import psutil +from django.core.management.base import BaseCommand +from django.db import connections +from django.utils import timezone +from django_simple_queue.models import Task class ManagedEventLoop: @@ -84,6 +86,14 @@ def handle(self, *args, **options): print(f"{timezone.now()}: [RAM Usage: {log_memory_usage()} MB] Heartbeat..") queued_task = Task.objects.filter(status=Task.QUEUED).order_by('modified').first() if queued_task: - process_task(task_id=queued_task.id) + # since parent connections are copied to the child process + # avoid corruption by closing all connections + connections.close_all() + + # Create a new process for the task + p = Process(target=process_task, args=(queued_task.id,)) + p.start() + p.join() # Wait for the process to complete + except KeyboardInterrupt: pass From 53e39495f7303a3603a55b3a3c2ef967467d235b Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Fri, 28 Mar 2025 17:29:40 +0100 Subject: [PATCH 03/14] [doc] Added documentation detailing the benefits of custom settings for taskworkers (cherry picked from commit 334e504f34227c53ca5f8256c86fe6f389e11a0b) --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/README.md b/README.md index 3225d2a..3ef204a 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,61 @@ path('django_simple_queue/', include('django_simple_queue.urls')), ## Usage +### Create a custom settings file `task_worker_settings.py` +If you want a lean taskworker process, thatdoes not run all of the django apps, create a custom settings file in the same +directory as `/settings.py` that imports the webserver settings, but doesn't load all the django apps that +the webserver needs. In our tests, this decreased the RAM usage for a single taskworker from `400-500 MB` to just `30-50 MB`[^1]. + +Also the number of subprocesses (as reported by htop) drops from `10-21` subprocesses to just `1`! + +Here is an example -- + +```python +# task_worker_settings.py +from .settings import * + +# Application definition - keep only what's needed for task processing +INSTALLED_APPS = [ + # django packages - minimal required + "django.contrib.contenttypes", # Required for DB models + "django_simple_queue", + # Only apps needed for task processing + "", +] + +# Remove all middleware +MIDDLEWARE = [] + +# Remove template settings - task workers don't need templates +TEMPLATES = [] + +# Disable static/media file handling +STATICFILES_DIRS = () +STATIC_URL = None +STATIC_ROOT = None +MEDIA_ROOT = None +MEDIA_URL = None + +# Disable unnecessary Django features +USE_I18N = False +USE_TZ = True # Keep timezone support if your tasks rely on it + +# Optimize database connections +DATABASES["default"]["CONN_MAX_AGE"] = None # Persistent connections +DATABASES["default"]["OPTIONS"] = { + "connect_timeout": 10, +} + +# Remove unnecessary authentication settings +AUTH_PASSWORD_VALIDATORS = [] + +# Disable admin +ADMIN_ENABLED = False + +# Disable URL configuration +ROOT_URLCONF = None +``` + Start the worker process as follows: ```` python manage.py task_worker @@ -31,3 +86,6 @@ create_task( ) ```` The task queue can be viewed at /django_simple_queue/task + +[^1]: The metric used is Resident Set Size from the `psutil` python module, which double counts shared libraries and is +slightly more than actual RAM Usage. From 528766ca255dae005b3387f9e0f458c4d2ecb287 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Thu, 4 Sep 2025 13:02:19 +0200 Subject: [PATCH 04/14] feat: implement database-level pessimistic locking for task queue concurrency --- README.md | 10 +++++++ .../management/commands/task_worker.py | 29 +++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ef204a..91e2be4 100644 --- a/README.md +++ b/README.md @@ -87,5 +87,15 @@ create_task( ```` The task queue can be viewed at /django_simple_queue/task +## Concurrency and locking + +To prevent multiple workers from picking the same task, the worker command (`django_simple_queue/management/commands/task_worker.py`) uses database-level pessimistic locking when claiming tasks: + +- It wraps the selection in a transaction and queries with `select_for_update(skip_locked=True)` to lock a single queued task row and skip any rows currently locked by another worker. +- Once a task is selected under the lock, the worker immediately marks it as `In progress` (`Task.PROGRESS`) within the same transaction. Only after claiming does it spawn a subprocess to execute the task. +- If the database backend does not support `skip_locked`, the code falls back to `select_for_update()` without the `skip_locked` argument. While this still provides row-level locking on supported backends, `skip_locked` offers better concurrency characteristics. + +Recommended backends: For robust concurrent processing with multiple workers, use a database that supports `SELECT ... FOR UPDATE SKIP LOCKED` (e.g., PostgreSQL). SQLite may not provide full locking semantics for this pattern; it is best suited for development or single-worker setups. + [^1]: The metric used is Resident Set Size from the `psutil` python module, which double counts shared libraries and is slightly more than actual RAM Usage. diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index f134ff3..6c7ea36 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -10,6 +10,8 @@ import psutil from django.core.management.base import BaseCommand from django.db import connections +from django.db import transaction +from django.db.utils import NotSupportedError from django.utils import timezone from django_simple_queue.models import Task @@ -36,7 +38,7 @@ def __exit__(self, exc_type, exc_value, exc_tb): def process_task(task_id): task_obj = Task.objects.get(id=task_id) print(f"Initiating task id: {task_id}") - if task_obj.status == Task.QUEUED: # One more extra check to make sure + if task_obj.status in (Task.QUEUED, Task.PROGRESS): # Proceed if already claimed by parent # In case event loop gets killed with ManagedEventLoop() as loop: try: @@ -45,7 +47,6 @@ def process_task(task_id): func = getattr(module, path[-1]) args = json.loads(task_obj.args) task_obj.output = "" - task_obj.status = Task.PROGRESS task_obj.save() if inspect.isgeneratorfunction(func): @@ -84,14 +85,32 @@ def handle(self, *args, **options): while True: time.sleep(sleep_interval) print(f"{timezone.now()}: [RAM Usage: {log_memory_usage()} MB] Heartbeat..") - queued_task = Task.objects.filter(status=Task.QUEUED).order_by('modified').first() - if queued_task: + task_id = None + # Use pessimistic locking to claim a single queued task + with transaction.atomic(): + try: + qs = Task.objects.select_for_update(skip_locked=True) + except NotSupportedError: + # Fallback for DBs without skip_locked support + qs = Task.objects.select_for_update() + + queued_task = ( + qs.filter(status=Task.QUEUED) + .order_by('modified') + .first() + ) + if queued_task: + queued_task.status = Task.PROGRESS # claim the task + queued_task.save(update_fields=["status", "modified"]) + task_id = queued_task.id + + if task_id: # since parent connections are copied to the child process # avoid corruption by closing all connections connections.close_all() # Create a new process for the task - p = Process(target=process_task, args=(queued_task.id,)) + p = Process(target=process_task, args=(task_id,)) p.start() p.join() # Wait for the process to complete From b18374625214add6739f069879f3e8b3aa9db9be Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Tue, 27 Jan 2026 15:27:25 +0530 Subject: [PATCH 05/14] fix: prevent stalled tasks and add missing psutil dependency - Add psutil to requirements.txt (used in task_worker.py for memory logging but was never declared as a dependency) - Fix TypeError in error handler when task.output is None: if an exception occurs before output is initialized (e.g. during module import), the except block crashed on `None += str`, leaving the task stuck in "In Progress" forever with no error output Co-Authored-By: Claude Opus 4.5 --- django_simple_queue/management/commands/task_worker.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 6c7ea36..0e4791c 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -60,7 +60,7 @@ def process_task(task_id): task_obj.status = Task.COMPLETED task_obj.save() except Exception as e: - task_obj.output += f"{repr(e)}\n\n{traceback.format_exc()}" + task_obj.output = (task_obj.output or "") + f"{repr(e)}\n\n{traceback.format_exc()}" task_obj.status = Task.FAILED task_obj.save() finally: diff --git a/requirements.txt b/requirements.txt index 8b75561..aaf9bca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ Django>=3.0.0 +psutil From 7e0449333272a028942d106780b5fe5df9962404 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 16:19:41 +0530 Subject: [PATCH 06/14] feat: add test infrastructure Set up runtests.py with in-memory SQLite and test_tasks.py with helper callables so subsequent commits can include tests. Co-Authored-By: Claude Opus 4.5 --- django_simple_queue/test_tasks.py | 23 +++++++++++++++++++++++ runtests.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 django_simple_queue/test_tasks.py create mode 100644 runtests.py diff --git a/django_simple_queue/test_tasks.py b/django_simple_queue/test_tasks.py new file mode 100644 index 0000000..819d840 --- /dev/null +++ b/django_simple_queue/test_tasks.py @@ -0,0 +1,23 @@ +def return_hello(**kwargs): + return "hello" + + +def gen_abc(**kwargs): + yield "a" + yield "b" + yield "c" + + +def raise_error(**kwargs): + raise ValueError("test error") + + +def print_and_return(**kwargs): + print("log line from stdout") + import logging + + logging.info("log line from logging") + import sys + + print("log line from stderr", file=sys.stderr) + return "result" diff --git a/runtests.py b/runtests.py new file mode 100644 index 0000000..565639d --- /dev/null +++ b/runtests.py @@ -0,0 +1,29 @@ +import sys + +import django +from django.conf import settings +from django.test.utils import get_runner + +if not settings.configured: + settings.configure( + DATABASES={ + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": ":memory:", + } + }, + INSTALLED_APPS=[ + "django.contrib.contenttypes", + "django.contrib.auth", + "django_simple_queue", + ], + DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + ) + +django.setup() + +if __name__ == "__main__": + TestRunner = get_runner(settings) + test_runner = TestRunner() + failures = test_runner.run_tests(["django_simple_queue"]) + sys.exit(bool(failures)) From 8ef33cc498a118944ab127907f4b1f4b68cc0647 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 16:21:50 +0530 Subject: [PATCH 07/14] refactor: extract worker, monitor, signals modules from task_worker.py Move ManagedEventLoop and process_task (renamed execute_task) into worker.py. Add signals.py with 5 lifecycle signals (stubs). Add monitor.py with orphan detection stubs. Slim task_worker.py to orchestration only. Include tests for execute_task. Co-Authored-By: Claude Opus 4.5 --- .../management/commands/task_worker.py | 76 +++---------------- django_simple_queue/monitor.py | 8 ++ django_simple_queue/signals.py | 7 ++ django_simple_queue/tests.py | 34 ++++++++- django_simple_queue/worker.py | 55 ++++++++++++++ 5 files changed, 112 insertions(+), 68 deletions(-) create mode 100644 django_simple_queue/monitor.py create mode 100644 django_simple_queue/signals.py create mode 100644 django_simple_queue/worker.py diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 0e4791c..93788bf 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -1,73 +1,16 @@ -import asyncio -import importlib -import inspect -import json import random import time -import traceback from multiprocessing import Process import psutil from django.core.management.base import BaseCommand -from django.db import connections -from django.db import transaction +from django.db import connections, transaction from django.db.utils import NotSupportedError from django.utils import timezone from django_simple_queue.models import Task - - -class ManagedEventLoop: - def __init__(self): - self.loop = None - - def __enter__(self): - try: - self.loop = asyncio.get_running_loop() - except RuntimeError: - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) - return self.loop - - def __exit__(self, exc_type, exc_value, exc_tb): - if self.loop is not None: - self.loop.close() - - - -def process_task(task_id): - task_obj = Task.objects.get(id=task_id) - print(f"Initiating task id: {task_id}") - if task_obj.status in (Task.QUEUED, Task.PROGRESS): # Proceed if already claimed by parent - # In case event loop gets killed - with ManagedEventLoop() as loop: - try: - path = task_obj.task.split('.') - module = importlib.import_module('.'.join(path[:-1])) - func = getattr(module, path[-1]) - args = json.loads(task_obj.args) - task_obj.output = "" - task_obj.save() - - if inspect.isgeneratorfunction(func): - for i in func(**args): - output = i - task_obj.output += output - task_obj.save() - else: - task_obj.output = func(**args) - task_obj.save() - task_obj.status = Task.COMPLETED - task_obj.save() - except Exception as e: - task_obj.output = (task_obj.output or "") + f"{repr(e)}\n\n{traceback.format_exc()}" - task_obj.status = Task.FAILED - task_obj.save() - finally: - print(f"Finished task id: {task_id}") - - - +from django_simple_queue.monitor import detect_orphaned_tasks, handle_subprocess_exit +from django_simple_queue.worker import execute_task def log_memory_usage(): @@ -76,15 +19,18 @@ def log_memory_usage(): mem_info = process.memory_info() return round(mem_info.rss / (1024 * 1024), 2) + class Command(BaseCommand): - help = 'Executes the enqueued tasks.' + help = "Executes the enqueued tasks." def handle(self, *args, **options): try: sleep_interval = random.randint(3, 9) while True: time.sleep(sleep_interval) - print(f"{timezone.now()}: [RAM Usage: {log_memory_usage()} MB] Heartbeat..") + print( + f"{timezone.now()}: [RAM Usage: {log_memory_usage()} MB] Heartbeat.." + ) task_id = None # Use pessimistic locking to claim a single queued task with transaction.atomic(): @@ -95,9 +41,7 @@ def handle(self, *args, **options): qs = Task.objects.select_for_update() queued_task = ( - qs.filter(status=Task.QUEUED) - .order_by('modified') - .first() + qs.filter(status=Task.QUEUED).order_by("modified").first() ) if queued_task: queued_task.status = Task.PROGRESS # claim the task @@ -110,7 +54,7 @@ def handle(self, *args, **options): connections.close_all() # Create a new process for the task - p = Process(target=process_task, args=(task_id,)) + p = Process(target=execute_task, args=(task_id,)) p.start() p.join() # Wait for the process to complete diff --git a/django_simple_queue/monitor.py b/django_simple_queue/monitor.py new file mode 100644 index 0000000..6c6953f --- /dev/null +++ b/django_simple_queue/monitor.py @@ -0,0 +1,8 @@ +def detect_orphaned_tasks(): + """Check for PROGRESS tasks with dead worker PIDs. Stub.""" + pass + + +def handle_subprocess_exit(task_id, exit_code): + """Handle non-zero subprocess exit codes. Stub.""" + pass diff --git a/django_simple_queue/signals.py b/django_simple_queue/signals.py new file mode 100644 index 0000000..8dc1d2a --- /dev/null +++ b/django_simple_queue/signals.py @@ -0,0 +1,7 @@ +import django.dispatch + +before_job = django.dispatch.Signal() # kwargs: task +on_success = django.dispatch.Signal() # kwargs: task +on_failure = django.dispatch.Signal() # kwargs: task, error +before_loop = django.dispatch.Signal() # kwargs: task, iteration +after_loop = django.dispatch.Signal() # kwargs: task, output, iteration diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index 7ce503c..fbf0fba 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -1,3 +1,33 @@ -from django.test import TestCase +from django.test import TransactionTestCase -# Create your tests here. +from django_simple_queue.models import Task +from django_simple_queue.worker import execute_task + + +class ExecuteTaskTest(TransactionTestCase): + def test_regular_task(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.status, Task.COMPLETED) + self.assertEqual(task.output, "hello") + + def test_generator_task(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.gen_abc", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.status, Task.COMPLETED) + self.assertEqual(task.output, "abc") + + def test_failing_task(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.raise_error", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.status, Task.FAILED) + self.assertIn("ValueError", task.output) diff --git a/django_simple_queue/worker.py b/django_simple_queue/worker.py new file mode 100644 index 0000000..83743a0 --- /dev/null +++ b/django_simple_queue/worker.py @@ -0,0 +1,55 @@ +import asyncio +import importlib +import inspect +import json +import traceback + +from django_simple_queue.models import Task + + +class ManagedEventLoop: + def __init__(self): + self.loop = None + + def __enter__(self): + try: + self.loop = asyncio.get_running_loop() + except RuntimeError: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + return self.loop + + def __exit__(self, exc_type, exc_value, exc_tb): + if self.loop is not None: + self.loop.close() + + +def execute_task(task_id, log_fd=None): + task_obj = Task.objects.get(id=task_id) + print(f"Initiating task id: {task_id}") + if task_obj.status in (Task.QUEUED, Task.PROGRESS): + with ManagedEventLoop(): + try: + path = task_obj.task.split(".") + module = importlib.import_module(".".join(path[:-1])) + func = getattr(module, path[-1]) + args = json.loads(task_obj.args) + task_obj.output = "" + task_obj.save() + + if inspect.isgeneratorfunction(func): + for i in func(**args): + output = i + task_obj.output += output + task_obj.save() + else: + task_obj.output = func(**args) + task_obj.save() + task_obj.status = Task.COMPLETED + task_obj.save() + except Exception as e: + task_obj.output = (task_obj.output or "") + f"{repr(e)}\n\n{traceback.format_exc()}" + task_obj.status = Task.FAILED + task_obj.save() + finally: + print(f"Finished task id: {task_id}") From 428cebc4cacb81fb862ba2d00ca90ea41c97ba49 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 16:24:01 +0530 Subject: [PATCH 08/14] feat: add worker_pid, error, log fields and separate output from error Add three new fields to Task model: worker_pid for tracking which process owns a task, error for storing tracebacks separately from output, and log for captured process output. Update execute_task to write errors to task.error instead of task.output. Co-Authored-By: Claude Opus 4.5 --- ...002_task_error_task_log_task_worker_pid.py | 28 +++++++++++++++ django_simple_queue/models.py | 6 ++++ django_simple_queue/tests.py | 34 ++++++++++++++++++- django_simple_queue/worker.py | 2 +- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 django_simple_queue/migrations/0002_task_error_task_log_task_worker_pid.py diff --git a/django_simple_queue/migrations/0002_task_error_task_log_task_worker_pid.py b/django_simple_queue/migrations/0002_task_error_task_log_task_worker_pid.py new file mode 100644 index 0000000..c0b834b --- /dev/null +++ b/django_simple_queue/migrations/0002_task_error_task_log_task_worker_pid.py @@ -0,0 +1,28 @@ +# Generated by Django 6.0.1 on 2026-01-28 10:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_simple_queue', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='error', + field=models.TextField(blank=True, null=True, verbose_name='Error'), + ), + migrations.AddField( + model_name='task', + name='log', + field=models.TextField(blank=True, null=True, verbose_name='Log'), + ), + migrations.AddField( + model_name='task', + name='worker_pid', + field=models.IntegerField(blank=True, null=True, verbose_name='Worker PID'), + ), + ] diff --git a/django_simple_queue/models.py b/django_simple_queue/models.py index 361dc21..cc70217 100644 --- a/django_simple_queue/models.py +++ b/django_simple_queue/models.py @@ -30,6 +30,9 @@ class Task(models.Model): args = models.TextField(_("Arguments"), null=True, blank=True, help_text="Arguments in JSON format") status = models.IntegerField(_("Status"), default=QUEUED, choices=STATUS_CHOICES) output = models.TextField(_("Output"), null=True, blank=True) + worker_pid = models.IntegerField(_("Worker PID"), null=True, blank=True) + error = models.TextField(_("Error"), null=True, blank=True) + log = models.TextField(_("Log"), null=True, blank=True) def __str__(self): return str(self.id) @@ -48,6 +51,9 @@ def as_dict(self): "args": self.args, "status": self.get_status_display(), "output": self.output, + "worker_pid": self.worker_pid, + "error": self.error, + "log": self.log, } @staticmethod diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index fbf0fba..0a8d176 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -30,4 +30,36 @@ def test_failing_task(self): execute_task(task.id) task.refresh_from_db() self.assertEqual(task.status, Task.FAILED) - self.assertIn("ValueError", task.output) + self.assertIn("ValueError", task.error) + + +class FieldSeparationTest(TransactionTestCase): + def test_output_only_has_return_value(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.output, "hello") + self.assertIsNone(task.error) + + def test_error_has_traceback(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.raise_error", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.status, Task.FAILED) + self.assertIn("ValueError", task.error) + self.assertIn("Traceback", task.error) + # output should have whatever was produced before the error + self.assertEqual(task.output, "") + + def test_generator_output_concatenation(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.gen_abc", args="{}" + ) + execute_task(task.id) + task.refresh_from_db() + self.assertEqual(task.output, "abc") + self.assertIsNone(task.error) diff --git a/django_simple_queue/worker.py b/django_simple_queue/worker.py index 83743a0..cddc73e 100644 --- a/django_simple_queue/worker.py +++ b/django_simple_queue/worker.py @@ -48,7 +48,7 @@ def execute_task(task_id, log_fd=None): task_obj.status = Task.COMPLETED task_obj.save() except Exception as e: - task_obj.output = (task_obj.output or "") + f"{repr(e)}\n\n{traceback.format_exc()}" + task_obj.error = f"{repr(e)}\n\n{traceback.format_exc()}" task_obj.status = Task.FAILED task_obj.save() finally: From a87d5f1fb576561203cd36c964605e13bd2316b2 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 16:26:43 +0530 Subject: [PATCH 09/14] feat: add worker PID tracking and orphan detection Set worker_pid on task claim, clear after subprocess completes. Implement detect_orphaned_tasks to find PROGRESS tasks with dead PIDs and mark them FAILED. Add handle_subprocess_exit to catch non-zero exit codes. Co-Authored-By: Claude Opus 4.5 --- .../management/commands/task_worker.py | 16 +++++- django_simple_queue/monitor.py | 42 ++++++++++++++-- django_simple_queue/tests.py | 50 +++++++++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 93788bf..cabf718 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -1,3 +1,4 @@ +import os import random import time from multiprocessing import Process @@ -45,7 +46,10 @@ def handle(self, *args, **options): ) if queued_task: queued_task.status = Task.PROGRESS # claim the task - queued_task.save(update_fields=["status", "modified"]) + queued_task.worker_pid = os.getpid() + queued_task.save( + update_fields=["status", "modified", "worker_pid"] + ) task_id = queued_task.id if task_id: @@ -58,5 +62,15 @@ def handle(self, *args, **options): p.start() p.join() # Wait for the process to complete + # Clear worker_pid (parent owns this field) + task = Task.objects.get(id=task_id) + task.worker_pid = None + task.save(update_fields=["worker_pid", "modified"]) + + handle_subprocess_exit(task_id, p.exitcode) + + # Check for orphaned tasks before polling for new ones + detect_orphaned_tasks() + except KeyboardInterrupt: pass diff --git a/django_simple_queue/monitor.py b/django_simple_queue/monitor.py index 6c6953f..ed54321 100644 --- a/django_simple_queue/monitor.py +++ b/django_simple_queue/monitor.py @@ -1,8 +1,42 @@ +import os + +from django.db import transaction + +from django_simple_queue import signals +from django_simple_queue.models import Task + + def detect_orphaned_tasks(): - """Check for PROGRESS tasks with dead worker PIDs. Stub.""" - pass + """Check for PROGRESS tasks with dead worker PIDs and mark them FAILED.""" + with transaction.atomic(): + in_progress = Task.objects.select_for_update(skip_locked=True).filter( + status=Task.PROGRESS, worker_pid__isnull=False + ) + for task in in_progress: + try: + os.kill(task.worker_pid, 0) + except ProcessLookupError: + task.error = (task.error or "") + ( + f"\nTask failed: worker process (PID {task.worker_pid}) no longer running" + ) + task.status = Task.FAILED + task.worker_pid = None + task.save( + update_fields=["status", "error", "worker_pid", "modified"] + ) + signals.on_failure.send(sender=Task, task=task, error=None) + except PermissionError: + pass # PID exists, different user — worker is alive def handle_subprocess_exit(task_id, exit_code): - """Handle non-zero subprocess exit codes. Stub.""" - pass + """Handle non-zero subprocess exit codes.""" + if exit_code is None or exit_code == 0: + return + task = Task.objects.get(id=task_id) + if task.status == Task.PROGRESS: + task.error = (task.error or "") + f"\nWorker subprocess exited with code {exit_code}" + task.status = Task.FAILED + task.worker_pid = None + task.save(update_fields=["status", "error", "worker_pid", "modified"]) + signals.on_failure.send(sender=Task, task=task, error=None) diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index 0a8d176..4828450 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -1,6 +1,9 @@ +import os + from django.test import TransactionTestCase from django_simple_queue.models import Task +from django_simple_queue.monitor import detect_orphaned_tasks, handle_subprocess_exit from django_simple_queue.worker import execute_task @@ -63,3 +66,50 @@ def test_generator_output_concatenation(self): task.refresh_from_db() self.assertEqual(task.output, "abc") self.assertIsNone(task.error) + + +class OrphanDetectionTest(TransactionTestCase): + def test_dead_pid_marks_task_failed(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", + args="{}", + status=Task.PROGRESS, + worker_pid=999999, # nonexistent PID + ) + detect_orphaned_tasks() + task.refresh_from_db() + self.assertEqual(task.status, Task.FAILED) + self.assertIn("no longer running", task.error) + self.assertIsNone(task.worker_pid) + + def test_live_pid_not_touched(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", + args="{}", + status=Task.PROGRESS, + worker_pid=os.getpid(), # this process is alive + ) + detect_orphaned_tasks() + task.refresh_from_db() + self.assertEqual(task.status, Task.PROGRESS) + + def test_nonzero_exit_code_marks_failed(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", + args="{}", + status=Task.PROGRESS, + ) + handle_subprocess_exit(task.id, exit_code=1) + task.refresh_from_db() + self.assertEqual(task.status, Task.FAILED) + self.assertIn("exited with code 1", task.error) + + def test_zero_exit_code_no_change(self): + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", + args="{}", + status=Task.COMPLETED, + ) + handle_subprocess_exit(task.id, exit_code=0) + task.refresh_from_db() + self.assertEqual(task.status, Task.COMPLETED) From c88b5c4a915faaa1cdf5125357dc37b781edac39 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 16:32:11 +0530 Subject: [PATCH 10/14] feat: fire lifecycle signals in execute_task Emit before_job, on_success, on_failure, before_loop, and after_loop signals at appropriate points in the task execution flow. Generator tasks fire before_loop/after_loop per iteration. Co-Authored-By: Claude Opus 4.5 --- django_simple_queue/tests.py | 79 +++++++++++++++++++++++++++++++++++ django_simple_queue/worker.py | 20 ++++++++- 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index 4828450..b007ebf 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -2,6 +2,7 @@ from django.test import TransactionTestCase +from django_simple_queue import signals from django_simple_queue.models import Task from django_simple_queue.monitor import detect_orphaned_tasks, handle_subprocess_exit from django_simple_queue.worker import execute_task @@ -113,3 +114,81 @@ def test_zero_exit_code_no_change(self): handle_subprocess_exit(task.id, exit_code=0) task.refresh_from_db() self.assertEqual(task.status, Task.COMPLETED) + + +class SignalTest(TransactionTestCase): + def test_regular_task_signals(self): + received = [] + + def on_before(sender, task, **kw): + received.append("before_job") + + def on_success(sender, task, **kw): + received.append("on_success") + + signals.before_job.connect(on_before) + signals.on_success.connect(on_success) + try: + task = Task.objects.create( + task="django_simple_queue.test_tasks.return_hello", args="{}" + ) + execute_task(task.id) + self.assertEqual(received, ["before_job", "on_success"]) + finally: + signals.before_job.disconnect(on_before) + signals.on_success.disconnect(on_success) + + def test_failing_task_signals(self): + received = [] + errors = [] + + def on_before(sender, task, **kw): + received.append("before_job") + + def on_fail(sender, task, error=None, **kw): + received.append("on_failure") + errors.append(error) + + signals.before_job.connect(on_before) + signals.on_failure.connect(on_fail) + try: + task = Task.objects.create( + task="django_simple_queue.test_tasks.raise_error", args="{}" + ) + execute_task(task.id) + self.assertEqual(received, ["before_job", "on_failure"]) + self.assertIsInstance(errors[0], ValueError) + finally: + signals.before_job.disconnect(on_before) + signals.on_failure.disconnect(on_fail) + + def test_generator_loop_signals(self): + iterations = [] + + def on_before_loop(sender, task, iteration, **kw): + iterations.append(("before", iteration)) + + def on_after_loop(sender, task, output, iteration, **kw): + iterations.append(("after", iteration, output)) + + signals.before_loop.connect(on_before_loop) + signals.after_loop.connect(on_after_loop) + try: + task = Task.objects.create( + task="django_simple_queue.test_tasks.gen_abc", args="{}" + ) + execute_task(task.id) + self.assertEqual( + iterations, + [ + ("before", 0), + ("after", 0, "a"), + ("before", 1), + ("after", 1, "b"), + ("before", 2), + ("after", 2, "c"), + ], + ) + finally: + signals.before_loop.disconnect(on_before_loop) + signals.after_loop.disconnect(on_after_loop) diff --git a/django_simple_queue/worker.py b/django_simple_queue/worker.py index cddc73e..6fe50ff 100644 --- a/django_simple_queue/worker.py +++ b/django_simple_queue/worker.py @@ -4,6 +4,7 @@ import json import traceback +from django_simple_queue import signals from django_simple_queue.models import Task @@ -29,6 +30,7 @@ def execute_task(task_id, log_fd=None): print(f"Initiating task id: {task_id}") if task_obj.status in (Task.QUEUED, Task.PROGRESS): with ManagedEventLoop(): + signals.before_job.send(sender=Task, task=task_obj) try: path = task_obj.task.split(".") module = importlib.import_module(".".join(path[:-1])) @@ -38,18 +40,32 @@ def execute_task(task_id, log_fd=None): task_obj.save() if inspect.isgeneratorfunction(func): - for i in func(**args): - output = i + gen = func(**args) + iteration = 0 + for output in gen: + signals.before_loop.send( + sender=Task, task=task_obj, iteration=iteration + ) task_obj.output += output task_obj.save() + signals.after_loop.send( + sender=Task, + task=task_obj, + output=output, + iteration=iteration, + ) + iteration += 1 else: task_obj.output = func(**args) task_obj.save() + task_obj.status = Task.COMPLETED task_obj.save() + signals.on_success.send(sender=Task, task=task_obj) except Exception as e: task_obj.error = f"{repr(e)}\n\n{traceback.format_exc()}" task_obj.status = Task.FAILED task_obj.save() + signals.on_failure.send(sender=Task, task=task_obj, error=e) finally: print(f"Finished task id: {task_id}") From 27747beec10a8ebbd4a53ffdc09996a387e43982 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 18:10:25 +0530 Subject: [PATCH 11/14] feat: add pipe-based log capture from child process Capture stdout, stderr, and Python logging output from the child process via os.pipe(). Parent creates pipe before fork, drains it in a background thread during p.join(), then stores the captured log in task.log. The worker.py execute_task() redirects sys.stdout, sys.stderr, and root logger to the pipe fd when log_fd is provided. Also update runtests.py to use file-based SQLite with matching TEST name so subprocess tests can access the same database. Co-Authored-By: Claude Opus 4.5 --- .../management/commands/task_worker.py | 33 +++++- django_simple_queue/tests.py | 42 +++++++ django_simple_queue/worker.py | 105 +++++++++++------- runtests.py | 15 ++- 4 files changed, 149 insertions(+), 46 deletions(-) diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index cabf718..14e5bc1 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -1,5 +1,6 @@ import os import random +import threading import time from multiprocessing import Process @@ -57,15 +58,37 @@ def handle(self, *args, **options): # avoid corruption by closing all connections connections.close_all() - # Create a new process for the task - p = Process(target=execute_task, args=(task_id,)) + # Create pipe for capturing child stdout/stderr/logging + read_fd, write_fd = os.pipe() + p = Process( + target=execute_task, args=(task_id, write_fd) + ) p.start() - p.join() # Wait for the process to complete + os.close(write_fd) # Parent doesn't write + + log_chunks = [] + + def drain(): + with os.fdopen(read_fd, "r", closefd=True) as f: + while True: + chunk = f.read(4096) + if not chunk: + break + log_chunks.append(chunk) - # Clear worker_pid (parent owns this field) + reader = threading.Thread(target=drain, daemon=True) + reader.start() + p.join() + reader.join(timeout=5) + + # Store log + clear PID (parent-owned fields only) + log_text = "".join(log_chunks) task = Task.objects.get(id=task_id) + task.log = log_text if log_text else None task.worker_pid = None - task.save(update_fields=["worker_pid", "modified"]) + task.save( + update_fields=["log", "worker_pid", "modified"] + ) handle_subprocess_exit(task_id, p.exitcode) diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index b007ebf..6fef23b 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -1,4 +1,6 @@ import os +import threading +from multiprocessing import Process from django.test import TransactionTestCase @@ -192,3 +194,43 @@ def on_after_loop(sender, task, output, iteration, **kw): finally: signals.before_loop.disconnect(on_before_loop) signals.after_loop.disconnect(on_after_loop) + + +class PipeLogCaptureTest(TransactionTestCase): + def test_stdout_captured_in_pipe(self): + from django.db import connections + + task = Task.objects.create( + task="django_simple_queue.test_tasks.print_and_return", + args="{}", + status=Task.PROGRESS, # Set to PROGRESS like parent would + ) + # Close parent's DB connections before fork (like task_worker does) + connections.close_all() + + read_fd, write_fd = os.pipe() + p = Process(target=execute_task, args=(task.id, write_fd)) + p.start() + os.close(write_fd) + + log_chunks = [] + + def drain(): + with os.fdopen(read_fd, "r", closefd=True) as f: + while True: + chunk = f.read(4096) + if not chunk: + break + log_chunks.append(chunk) + + reader = threading.Thread(target=drain, daemon=True) + reader.start() + p.join() + reader.join(timeout=5) + + log_output = "".join(log_chunks) + self.assertIn("log line from stdout", log_output) + self.assertIn("log line from stderr", log_output) + self.assertIn("log line from logging", log_output) + task.refresh_from_db() + self.assertEqual(task.output, "result") # output is clean diff --git a/django_simple_queue/worker.py b/django_simple_queue/worker.py index 6fe50ff..af7c82a 100644 --- a/django_simple_queue/worker.py +++ b/django_simple_queue/worker.py @@ -2,6 +2,9 @@ import importlib import inspect import json +import logging +import os +import sys import traceback from django_simple_queue import signals @@ -26,46 +29,68 @@ def __exit__(self, exc_type, exc_value, exc_tb): def execute_task(task_id, log_fd=None): - task_obj = Task.objects.get(id=task_id) - print(f"Initiating task id: {task_id}") - if task_obj.status in (Task.QUEUED, Task.PROGRESS): - with ManagedEventLoop(): - signals.before_job.send(sender=Task, task=task_obj) - try: - path = task_obj.task.split(".") - module = importlib.import_module(".".join(path[:-1])) - func = getattr(module, path[-1]) - args = json.loads(task_obj.args) - task_obj.output = "" - task_obj.save() + log_file = None + log_handler = None + if log_fd is not None: + log_file = os.fdopen(log_fd, "w") + sys.stdout = log_file + sys.stderr = log_file + log_handler = logging.StreamHandler(log_file) + log_handler.setFormatter( + logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s") + ) + logging.root.addHandler(log_handler) + logging.root.setLevel(logging.DEBUG) - if inspect.isgeneratorfunction(func): - gen = func(**args) - iteration = 0 - for output in gen: - signals.before_loop.send( - sender=Task, task=task_obj, iteration=iteration - ) - task_obj.output += output - task_obj.save() - signals.after_loop.send( - sender=Task, - task=task_obj, - output=output, - iteration=iteration, - ) - iteration += 1 - else: - task_obj.output = func(**args) + try: + task_obj = Task.objects.get(id=task_id) + print(f"Initiating task id: {task_id}") + if task_obj.status in (Task.QUEUED, Task.PROGRESS): + with ManagedEventLoop(): + signals.before_job.send(sender=Task, task=task_obj) + try: + path = task_obj.task.split(".") + module = importlib.import_module(".".join(path[:-1])) + func = getattr(module, path[-1]) + args = json.loads(task_obj.args) + task_obj.output = "" task_obj.save() - task_obj.status = Task.COMPLETED - task_obj.save() - signals.on_success.send(sender=Task, task=task_obj) - except Exception as e: - task_obj.error = f"{repr(e)}\n\n{traceback.format_exc()}" - task_obj.status = Task.FAILED - task_obj.save() - signals.on_failure.send(sender=Task, task=task_obj, error=e) - finally: - print(f"Finished task id: {task_id}") + if inspect.isgeneratorfunction(func): + gen = func(**args) + iteration = 0 + for output in gen: + signals.before_loop.send( + sender=Task, task=task_obj, iteration=iteration + ) + task_obj.output += output + task_obj.save() + signals.after_loop.send( + sender=Task, + task=task_obj, + output=output, + iteration=iteration, + ) + iteration += 1 + else: + task_obj.output = func(**args) + task_obj.save() + + task_obj.status = Task.COMPLETED + task_obj.save() + signals.on_success.send(sender=Task, task=task_obj) + except Exception as e: + task_obj.error = f"{repr(e)}\n\n{traceback.format_exc()}" + task_obj.status = Task.FAILED + task_obj.save() + signals.on_failure.send(sender=Task, task=task_obj, error=e) + finally: + print(f"Finished task id: {task_id}") + finally: + if log_file is not None: + if log_handler is not None: + logging.root.removeHandler(log_handler) + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ + log_file.flush() + log_file.close() diff --git a/runtests.py b/runtests.py index 565639d..bb4825a 100644 --- a/runtests.py +++ b/runtests.py @@ -1,15 +1,25 @@ +import os import sys +import tempfile import django from django.conf import settings from django.test.utils import get_runner +# Use a file-based SQLite database so subprocesses can access it +# (in-memory databases are not shared across processes) +TEST_DB_FILE = os.path.join(tempfile.gettempdir(), "django_simple_queue_test.db") + if not settings.configured: settings.configure( DATABASES={ "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": ":memory:", + "NAME": TEST_DB_FILE, + # Use same name for test DB so subprocesses can access it + "TEST": { + "NAME": TEST_DB_FILE, + }, } }, INSTALLED_APPS=[ @@ -26,4 +36,7 @@ TestRunner = get_runner(settings) test_runner = TestRunner() failures = test_runner.run_tests(["django_simple_queue"]) + # Clean up test database file + if os.path.exists(TEST_DB_FILE): + os.remove(TEST_DB_FILE) sys.exit(bool(failures)) From d91b02bbcd33ebfbc291c968ec24727b7900474c Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 23:30:12 +0530 Subject: [PATCH 12/14] feat: add security hardening and task timeout - Fix XSS: replace inline HTML with Django template (auto-escaping) - Fix bare except clauses with specific exception types - Replace mark_safe with format_html in admin - Add optional task allowlist (DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS) - Add task execution timeout (DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT) - Add conf.py settings module - Add security and timeout tests Co-Authored-By: Claude Opus 4.5 --- django_simple_queue/admin.py | 12 +- django_simple_queue/conf.py | 85 ++++++++ .../management/commands/task_worker.py | 47 +++- django_simple_queue/models.py | 22 +- django_simple_queue/monitor.py | 15 ++ .../django_simple_queue/task_status.html | 52 +++++ django_simple_queue/test_tasks.py | 8 + django_simple_queue/tests.py | 206 +++++++++++++++++- django_simple_queue/utils.py | 53 ++++- django_simple_queue/views.py | 36 ++- runtests.py | 9 + 11 files changed, 494 insertions(+), 51 deletions(-) create mode 100644 django_simple_queue/conf.py create mode 100644 django_simple_queue/templates/django_simple_queue/task_status.html diff --git a/django_simple_queue/admin.py b/django_simple_queue/admin.py index b854370..ed22234 100644 --- a/django_simple_queue/admin.py +++ b/django_simple_queue/admin.py @@ -1,7 +1,6 @@ -from django.contrib import admin -from django.shortcuts import reverse -from django.utils.safestring import mark_safe -from django.contrib import messages +from django.contrib import admin, messages +from django.urls import reverse +from django.utils.html import format_html from django.utils.translation import ngettext from django_simple_queue.models import Task @@ -15,11 +14,12 @@ def get_readonly_fields(self, request, obj=None): return self.readonly_fields def status_page_link(self, obj): - return mark_safe("{}".format( + return format_html( + '{}', reverse('django_simple_queue:task'), obj.id, obj.get_status_display(), - )) + ) status_page_link.short_description = "Status" @admin.action(description='Enqueue') diff --git a/django_simple_queue/conf.py b/django_simple_queue/conf.py new file mode 100644 index 0000000..ac0437d --- /dev/null +++ b/django_simple_queue/conf.py @@ -0,0 +1,85 @@ +""" +Configuration settings for django_simple_queue. + +Settings are read from Django's settings.py with the DJANGO_SIMPLE_QUEUE_ prefix. +""" +from django.conf import settings + + +def get_allowed_tasks(): + """ + Returns the set of allowed task callables. + + Configure in settings.py: + DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = { + "myapp.tasks.process_order", + "myapp.tasks.send_email", + } + + If not set or set to None, ALL tasks are allowed (unsafe, but backwards-compatible). + Set to an empty set to disallow all tasks. + """ + allowed = getattr(settings, "DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS", None) + if allowed is None: + return None # No restriction (backwards-compatible but unsafe) + return set(allowed) + + +def is_task_allowed(task_path): + """ + Check if a task path is in the allowed list. + + Args: + task_path: Dotted path to the callable (e.g., "myapp.tasks.process_order") + + Returns: + True if allowed, False if not allowed. + If DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS is not configured, returns True (permissive). + """ + allowed = get_allowed_tasks() + if allowed is None: + # No allowlist configured - permissive mode (backwards-compatible) + return True + return task_path in allowed + + +def get_max_output_size(): + """ + Returns the maximum allowed output size in bytes. + + Configure in settings.py: + DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE = 1_000_000 # 1MB + + Default: 10MB + """ + return getattr(settings, "DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE", 10 * 1024 * 1024) + + +def get_max_args_size(): + """ + Returns the maximum allowed args JSON size in bytes. + + Configure in settings.py: + DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE = 100_000 # 100KB + + Default: 1MB + """ + return getattr(settings, "DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE", 1024 * 1024) + + +def get_task_timeout(): + """ + Returns the maximum execution time for a task in seconds. + + Configure in settings.py: + DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = 300 # 5 minutes + + If not set or set to None, tasks can run indefinitely. + Set to 0 or negative to disable timeout. + + Default: 3600 (1 hour) + """ + timeout = getattr(settings, "DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT", 3600) + if timeout is None or timeout <= 0: + return None + return timeout diff --git a/django_simple_queue/management/commands/task_worker.py b/django_simple_queue/management/commands/task_worker.py index 14e5bc1..3c8ca8e 100644 --- a/django_simple_queue/management/commands/task_worker.py +++ b/django_simple_queue/management/commands/task_worker.py @@ -10,8 +10,13 @@ from django.db.utils import NotSupportedError from django.utils import timezone +from django_simple_queue.conf import get_task_timeout from django_simple_queue.models import Task -from django_simple_queue.monitor import detect_orphaned_tasks, handle_subprocess_exit +from django_simple_queue.monitor import ( + detect_orphaned_tasks, + handle_subprocess_exit, + handle_task_timeout, +) from django_simple_queue.worker import execute_task @@ -28,6 +33,12 @@ class Command(BaseCommand): def handle(self, *args, **options): try: sleep_interval = random.randint(3, 9) + timeout = get_task_timeout() + if timeout: + print(f"Task timeout configured: {timeout} seconds") + else: + print("Task timeout: disabled (tasks can run indefinitely)") + while True: time.sleep(sleep_interval) print( @@ -60,9 +71,7 @@ def handle(self, *args, **options): # Create pipe for capturing child stdout/stderr/logging read_fd, write_fd = os.pipe() - p = Process( - target=execute_task, args=(task_id, write_fd) - ) + p = Process(target=execute_task, args=(task_id, write_fd)) p.start() os.close(write_fd) # Parent doesn't write @@ -78,7 +87,26 @@ def drain(): reader = threading.Thread(target=drain, daemon=True) reader.start() - p.join() + + # Wait for process with optional timeout + p.join(timeout=timeout) + + timed_out = False + if p.is_alive(): + # Process is still running after timeout - terminate it + timed_out = True + print( + f"Task {task_id} timed out after {timeout}s, terminating..." + ) + p.terminate() + p.join(timeout=5) # Give it 5 seconds to terminate gracefully + + if p.is_alive(): + # Still alive? Force kill + print(f"Task {task_id} did not terminate, killing...") + p.kill() + p.join(timeout=2) + reader.join(timeout=5) # Store log + clear PID (parent-owned fields only) @@ -86,11 +114,12 @@ def drain(): task = Task.objects.get(id=task_id) task.log = log_text if log_text else None task.worker_pid = None - task.save( - update_fields=["log", "worker_pid", "modified"] - ) + task.save(update_fields=["log", "worker_pid", "modified"]) - handle_subprocess_exit(task_id, p.exitcode) + if timed_out: + handle_task_timeout(task_id, timeout) + else: + handle_subprocess_exit(task_id, p.exitcode) # Check for orphaned tasks before polling for new ones detect_orphaned_tasks() diff --git a/django_simple_queue/models.py b/django_simple_queue/models.py index cc70217..0f89c26 100644 --- a/django_simple_queue/models.py +++ b/django_simple_queue/models.py @@ -70,15 +70,27 @@ def clean_task(self): """Custom validation of the task field.""" try: self._callable_task(self.task) - except: + except (ImportError, AttributeError, TypeError, ValueError) as e: raise ValidationError({ - 'callable': ValidationError( - _('Invalid callable, must be importable'), code='invalid') + 'task': ValidationError( + _('Invalid callable: %(error)s'), + code='invalid', + params={'error': str(e)} + ) }) def clean_args(self): """Custom validation of the args field.""" + if self.args is None or self.args == "": + return + try: json.loads(self.args) - except: - raise ValidationError(_('Invalid JSON text'), code='invalid') + except (json.JSONDecodeError, TypeError, ValueError) as e: + raise ValidationError({ + 'args': ValidationError( + _('Invalid JSON: %(error)s'), + code='invalid', + params={'error': str(e)} + ) + }) diff --git a/django_simple_queue/monitor.py b/django_simple_queue/monitor.py index ed54321..f55217c 100644 --- a/django_simple_queue/monitor.py +++ b/django_simple_queue/monitor.py @@ -40,3 +40,18 @@ def handle_subprocess_exit(task_id, exit_code): task.worker_pid = None task.save(update_fields=["status", "error", "worker_pid", "modified"]) signals.on_failure.send(sender=Task, task=task, error=None) + + +def handle_task_timeout(task_id, timeout_seconds): + """Mark a task as failed due to exceeding the timeout.""" + task = Task.objects.get(id=task_id) + if task.status == Task.PROGRESS: + task.error = (task.error or "") + ( + f"\nTask timed out after {timeout_seconds} seconds" + ) + task.status = Task.FAILED + task.worker_pid = None + task.save(update_fields=["status", "error", "worker_pid", "modified"]) + signals.on_failure.send(sender=Task, task=task, error=TimeoutError( + f"Task exceeded {timeout_seconds}s timeout" + )) diff --git a/django_simple_queue/templates/django_simple_queue/task_status.html b/django_simple_queue/templates/django_simple_queue/task_status.html new file mode 100644 index 0000000..1a34ed8 --- /dev/null +++ b/django_simple_queue/templates/django_simple_queue/task_status.html @@ -0,0 +1,52 @@ + + + + + Task Status: {{ task.id }} + + + +

Task Status

+ + + + + + + + + + + + +
ID{{ task.id }}
Name{{ task.task }}
Arguments{{ task.args }}
Status + {{ task.get_status_display }} +
Created{{ task.created }}
Modified{{ task.modified }}
+ + {% if task.output %} +

Output

+ 
{{ task.output }}
 + {% endif %} + + {% if task.error %} +

Error

+ 
{{ task.error }}
 + {% endif %} + + {% if task.log %} +

Log

+ 
{{ task.log }}
 + {% endif %} + + diff --git a/django_simple_queue/test_tasks.py b/django_simple_queue/test_tasks.py index 819d840..02e197e 100644 --- a/django_simple_queue/test_tasks.py +++ b/django_simple_queue/test_tasks.py @@ -21,3 +21,11 @@ def print_and_return(**kwargs): print("log line from stderr", file=sys.stderr) return "result" + + +def sleep_task(seconds=1, **kwargs): + """Task that sleeps for testing timeout functionality.""" + import time + + time.sleep(seconds) + return f"slept for {seconds} seconds" diff --git a/django_simple_queue/tests.py b/django_simple_queue/tests.py index 6fef23b..3db4b64 100644 --- a/django_simple_queue/tests.py +++ b/django_simple_queue/tests.py @@ -2,11 +2,17 @@ import threading from multiprocessing import Process -from django.test import TransactionTestCase +from django.core.exceptions import ValidationError +from django.test import Client, TestCase, TransactionTestCase, override_settings from django_simple_queue import signals from django_simple_queue.models import Task -from django_simple_queue.monitor import detect_orphaned_tasks, handle_subprocess_exit +from django_simple_queue.monitor import ( + detect_orphaned_tasks, + handle_subprocess_exit, + handle_task_timeout, +) +from django_simple_queue.utils import TaskNotAllowedError, create_task from django_simple_queue.worker import execute_task @@ -117,6 +123,19 @@ def test_zero_exit_code_no_change(self): task.refresh_from_db() self.assertEqual(task.status, Task.COMPLETED) + def test_timeout_marks_task_failed(self): + """Tasks that timeout should be marked as failed.""" + task = Task.objects.create( + task="django_simple_queue.test_tasks.sleep_task", + args='{"seconds": 10}', + status=Task.PROGRESS, + ) + handle_task_timeout(task.id, timeout_seconds=5) + task.refresh_from_db() + self.assertEqual(task.status, Task.FAILED) + self.assertIn("timed out", task.error) + self.assertIn("5 seconds", task.error) + class SignalTest(TransactionTestCase): def test_regular_task_signals(self): @@ -234,3 +253,186 @@ def drain(): self.assertIn("log line from logging", log_output) task.refresh_from_db() self.assertEqual(task.output, "result") # output is clean + + +# ============================================================================= +# Security Tests +# ============================================================================= + + +class XSSProtectionTest(TestCase): + """Test that user-controlled data is properly escaped in HTML output.""" + + def setUp(self): + self.client = Client() + + def test_html_response_escapes_task_name(self): + """Task name with HTML should be escaped.""" + malicious_task = "" + task = Task.objects.create( + task=malicious_task, + args="{}", + status=Task.COMPLETED, + ) + response = self.client.get(f"/task?task_id={task.id}") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + # The script tag should be escaped, not rendered as HTML + self.assertNotIn(""}', + status=Task.QUEUED, + ) + response = self.client.get(f"/task?task_id={task.id}") + self.assertEqual(response.status_code, 200) + content = response.content.decode() + self.assertNotIn("", content) + + def test_json_response_does_not_escape(self): + """JSON response should contain raw data (not HTML-escaped).""" + task = Task.objects.create( + task="test.task", + args='{"key": ""}', + status=Task.QUEUED, + ) + response = self.client.get(f"/task?task_id={task.id}&type=json") + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "application/json") + # JSON should have raw data + self.assertIn("", response.content.decode()) + + +class TaskAllowlistTest(TestCase): + """Test task allowlist functionality.""" + + def test_no_allowlist_permits_all_tasks(self): + """Without DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS, all tasks are allowed.""" + # Default config has no allowlist + task_id = create_task( + task="django_simple_queue.test_tasks.return_hello", + args={} + ) + self.assertIsNotNone(task_id) + + @override_settings(DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS={ + "django_simple_queue.test_tasks.return_hello", + }) + def test_allowlist_permits_listed_tasks(self): + """Tasks in the allowlist should be permitted.""" + task_id = create_task( + task="django_simple_queue.test_tasks.return_hello", + args={} + ) + self.assertIsNotNone(task_id) + + @override_settings(DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS={ + "django_simple_queue.test_tasks.return_hello", + }) + def test_allowlist_blocks_unlisted_tasks(self): + """Tasks NOT in the allowlist should raise TaskNotAllowedError.""" + with self.assertRaises(TaskNotAllowedError) as ctx: + create_task( + task="django_simple_queue.test_tasks.gen_abc", + args={} + ) + self.assertIn("not in the allowed list", str(ctx.exception)) + + @override_settings(DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS=set()) + def test_empty_allowlist_blocks_all_tasks(self): + """Empty allowlist should block all tasks.""" + with self.assertRaises(TaskNotAllowedError): + create_task( + task="django_simple_queue.test_tasks.return_hello", + args={} + ) + + +class ModelValidationTest(TestCase): + """Test that model validation catches specific exceptions.""" + + def test_clean_args_rejects_invalid_json(self): + """Invalid JSON should raise ValidationError.""" + task = Task( + task="django_simple_queue.test_tasks.return_hello", + args="not valid json {", + ) + with self.assertRaises(ValidationError) as ctx: + task.clean_args() + self.assertIn("args", ctx.exception.message_dict) + + def test_clean_args_accepts_valid_json(self): + """Valid JSON should pass validation.""" + task = Task( + task="django_simple_queue.test_tasks.return_hello", + args='{"key": "value", "num": 123}', + ) + # Should not raise + task.clean_args() + + def test_clean_args_accepts_empty(self): + """Empty args should pass validation.""" + task = Task( + task="django_simple_queue.test_tasks.return_hello", + args=None, + ) + task.clean_args() # Should not raise + + task.args = "" + task.clean_args() # Should not raise + + def test_clean_task_rejects_nonexistent_module(self): + """Non-existent module should raise ValidationError.""" + task = Task( + task="nonexistent.module.function", + args="{}", + ) + with self.assertRaises(ValidationError) as ctx: + task.clean_task() + self.assertIn("task", ctx.exception.message_dict) + + +class ViewErrorHandlingTest(TestCase): + """Test proper error handling in views.""" + + def setUp(self): + self.client = Client() + + def test_missing_task_id_returns_400(self): + """Missing task_id parameter should return 400.""" + response = self.client.get("/task") + self.assertEqual(response.status_code, 400) + self.assertIn("Missing", response.content.decode()) + + def test_invalid_uuid_returns_400(self): + """Invalid UUID format should return 400.""" + response = self.client.get("/task?task_id=not-a-uuid") + self.assertEqual(response.status_code, 400) + self.assertIn("Invalid", response.content.decode()) + + def test_nonexistent_task_returns_400(self): + """Non-existent task UUID should return 400.""" + import uuid + fake_id = uuid.uuid4() + response = self.client.get(f"/task?task_id={fake_id}") + self.assertEqual(response.status_code, 400) + self.assertIn("not found", response.content.decode()) diff --git a/django_simple_queue/utils.py b/django_simple_queue/utils.py index a790554..eaec65d 100644 --- a/django_simple_queue/utils.py +++ b/django_simple_queue/utils.py @@ -1,13 +1,50 @@ -from django_simple_queue.models import Task import json +from django_simple_queue.conf import is_task_allowed, get_allowed_tasks +from django_simple_queue.models import Task + + +class TaskNotAllowedError(Exception): + """Raised when attempting to create a task that is not in the allowed list.""" + pass + def create_task(task, args): - if isinstance(args, dict): - obj = Task.objects.create( - task=task, - args=json.dumps(args) + """ + Create a new task to be executed by the worker. + + Args: + task: Dotted path to the callable (e.g., "myapp.tasks.process_order") + args: Dictionary of keyword arguments to pass to the callable + + Returns: + UUID of the created task + + Raises: + TypeError: If args is not a dict + TaskNotAllowedError: If task is not in DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS + + Example: + from django_simple_queue.utils import create_task + + task_id = create_task( + task="myapp.tasks.send_email", + args={"to": "user@example.com", "subject": "Hello"} ) - return obj.id - else: - raise TypeError("args should be of type dict.") \ No newline at end of file + """ + if not isinstance(args, dict): + raise TypeError("args should be of type dict.") + + if not is_task_allowed(task): + allowed = get_allowed_tasks() + if allowed is not None: + raise TaskNotAllowedError( + f"Task '{task}' is not in the allowed list. " + f"Add it to DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS in settings.py" + ) + + obj = Task.objects.create( + task=task, + args=json.dumps(args) + ) + return obj.id diff --git a/django_simple_queue/views.py b/django_simple_queue/views.py index a3394a9..fe71e84 100644 --- a/django_simple_queue/views.py +++ b/django_simple_queue/views.py @@ -1,30 +1,24 @@ +from django.core.exceptions import ValidationError +from django.http import JsonResponse, HttpResponseBadRequest from django.shortcuts import render -from django.http import JsonResponse, HttpResponse, HttpResponseBadRequest + from django_simple_queue.models import Task def view_task_status(request): """View for displaying the status of the task.""" + task_id = request.GET.get("task_id") + if not task_id: + return HttpResponseBadRequest("Missing task_id parameter.") + try: - task = Task.objects.get(id=request.GET.get("task_id")) - if request.GET.get("type") == "json": - return JsonResponse(task.as_dict) - html_message = f""" - - - - - - - -
Name{task.task}
Arguments{task.args}
Status{task.get_status_display()}
- 
{task.output}
 - - - """ - return HttpResponse(html_message) - except: - pass - return HttpResponseBadRequest("Invalid task_id requested.") + task = Task.objects.get(id=task_id) + except Task.DoesNotExist: + return HttpResponseBadRequest("Task not found.") + except (ValueError, TypeError, ValidationError): + return HttpResponseBadRequest("Invalid task_id format.") + if request.GET.get("type") == "json": + return JsonResponse(task.as_dict) + return render(request, "django_simple_queue/task_status.html", {"task": task}) diff --git a/runtests.py b/runtests.py index bb4825a..3efa75c 100644 --- a/runtests.py +++ b/runtests.py @@ -27,7 +27,16 @@ "django.contrib.auth", "django_simple_queue", ], + TEMPLATES=[ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "APP_DIRS": True, + }, + ], + ROOT_URLCONF="django_simple_queue.urls", DEFAULT_AUTO_FIELD="django.db.models.BigAutoField", + # For security tests - allowlist is NOT set by default (backwards-compatible) + # Tests can override this with @override_settings ) django.setup() From 6035e07ed610e38cee164f74b801832594be818f Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 23:36:21 +0530 Subject: [PATCH 13/14] docs: add MkDocs documentation with API reference from docstrings Set up MkDocs Material with mkdocstrings to generate documentation from Google-style docstrings. Adds comprehensive docstrings to all modules (models, signals, monitor, views, worker, admin), 17 documentation pages covering getting started, guides, API reference, and advanced topics, plus a GitHub Actions workflow for automatic GitHub Pages deployment. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/docs.yml | 52 +++++ .gitignore | 3 + django_simple_queue/admin.py | 41 ++++ django_simple_queue/models.py | 79 ++++++- django_simple_queue/monitor.py | 46 +++- django_simple_queue/signals.py | 64 ++++- django_simple_queue/urls.py | 2 +- django_simple_queue/views.py | 22 +- django_simple_queue/worker.py | 38 +++ docs/advanced/admin.md | 188 +++++++++++++++ docs/advanced/databases.md | 239 +++++++++++++++++++ docs/getting-started/configuration.md | 111 +++++++++ docs/getting-started/installation.md | 98 ++++++++ docs/getting-started/quickstart.md | 179 ++++++++++++++ docs/guides/creating-tasks.md | 240 +++++++++++++++++++ docs/guides/errors.md | 325 ++++++++++++++++++++++++++ docs/guides/generators.md | 238 +++++++++++++++++++ docs/guides/lifecycle.md | 202 ++++++++++++++++ docs/guides/signals.md | 246 +++++++++++++++++++ docs/guides/worker-optimization.md | 295 +++++++++++++++++++++++ docs/index.md | 65 ++++++ docs/reference/config.md | 109 +++++++++ docs/reference/models.md | 101 ++++++++ docs/reference/monitor.md | 116 +++++++++ docs/reference/signals.md | 112 +++++++++ docs/reference/utils.md | 120 ++++++++++ mkdocs.yml | 87 +++++++ 27 files changed, 3404 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/advanced/admin.md create mode 100644 docs/advanced/databases.md create mode 100644 docs/getting-started/configuration.md create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/guides/creating-tasks.md create mode 100644 docs/guides/errors.md create mode 100644 docs/guides/generators.md create mode 100644 docs/guides/lifecycle.md create mode 100644 docs/guides/signals.md create mode 100644 docs/guides/worker-optimization.md create mode 100644 docs/index.md create mode 100644 docs/reference/config.md create mode 100644 docs/reference/models.md create mode 100644 docs/reference/monitor.md create mode 100644 docs/reference/signals.md create mode 100644 docs/reference/utils.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..e5b8d63 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,52 @@ +name: Deploy Documentation + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + pip install mkdocs-material mkdocstrings[python] + pip install -e . + + - name: Build documentation + run: mkdocs build --strict + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 8f55b10..470417f 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ config.ini *.asc *.sqlite3 + +# MkDocs build output +site/ diff --git a/django_simple_queue/admin.py b/django_simple_queue/admin.py index ed22234..c63ebf4 100644 --- a/django_simple_queue/admin.py +++ b/django_simple_queue/admin.py @@ -1,3 +1,8 @@ +""" +Django admin configuration for django-simple-queue. + +Provides a customized admin interface for viewing and managing tasks. +""" from django.contrib import admin, messages from django.urls import reverse from django.utils.html import format_html @@ -7,6 +12,21 @@ @admin.register(Task) class TaskAdmin(admin.ModelAdmin): + """ + Admin interface for Task model. + + Features: + - All fields are read-only when editing an existing task + - Status column links to the task status page + - "Enqueue" action to re-queue selected tasks + - Search by task ID, callable path, and output + - Filter by status, created, and modified dates + + Note: + Tasks are generally created programmatically via ``create_task()``, + but the admin interface is useful for monitoring and re-queuing + failed tasks. + """ def get_readonly_fields(self, request, obj=None): if obj: @@ -14,6 +34,17 @@ def get_readonly_fields(self, request, obj=None): return self.readonly_fields def status_page_link(self, obj): + """ + Generate a clickable link to the task status page. + + Used as a column in the admin list view. Opens in a new tab. + + Args: + obj: The Task instance. + + Returns: + Safe HTML link to the task status view. + """ return format_html( '{}', reverse('django_simple_queue:task'), @@ -24,6 +55,16 @@ def status_page_link(self, obj): @admin.action(description='Enqueue') def enqueue_tasks(self, request, queryset): + """ + Admin action to re-queue selected tasks. + + Changes the status of selected tasks back to QUEUED so they will + be picked up by a worker. Useful for retrying failed tasks. + + Args: + request: The HTTP request. + queryset: QuerySet of selected Task instances. + """ updated = queryset.update(status=Task.QUEUED) self.message_user(request, ngettext( '%d task was successfully enqueued.', diff --git a/django_simple_queue/models.py b/django_simple_queue/models.py index 0f89c26..42973a5 100644 --- a/django_simple_queue/models.py +++ b/django_simple_queue/models.py @@ -1,3 +1,9 @@ +""" +Task model for django-simple-queue. + +This module defines the Task model which represents a unit of work to be +executed asynchronously by a worker process. +""" from django.db import models from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ @@ -7,7 +13,35 @@ class Task(models.Model): - """Model for the task.""" + """ + Represents a task to be executed asynchronously by the worker. + + A Task stores all the information needed to execute a callable function + with the specified arguments, along with its execution state and results. + + Attributes: + id: UUID primary key for the task. + created: Timestamp when the task was created. + modified: Timestamp when the task was last modified. + task: Dotted path to the callable (e.g., "myapp.tasks.send_email"). + args: JSON-serialized keyword arguments for the callable. + status: Current execution status (QUEUED, PROGRESS, COMPLETED, FAILED, CANCELLED). + output: Return value from the callable (stored as text). + worker_pid: Process ID of the worker handling this task. + error: Error message and traceback if the task failed. + log: Captured stdout/stderr/logging output from task execution. + + Example: + Creating a task directly (prefer using ``create_task`` utility):: + + from django_simple_queue.models import Task + import json + + task = Task.objects.create( + task="myapp.tasks.send_email", + args=json.dumps({"to": "user@example.com", "subject": "Hello"}) + ) + """ QUEUED = 0 PROGRESS = 1 @@ -43,6 +77,14 @@ class Meta: @property def as_dict(self): + """ + Returns a dictionary representation of the task. + + Useful for JSON serialization in API responses. + + Returns: + dict: Task data with string-formatted dates and status display. + """ return { "id": str(self.id), "created": str(self.created), @@ -58,7 +100,20 @@ def as_dict(self): @staticmethod def _callable_task(task): - """Checks if the task is callable.""" + """ + Validates and returns the callable for a task path. + + Args: + task: Dotted path to the callable (e.g., "myapp.tasks.send_email"). + + Returns: + The callable function or class. + + Raises: + ImportError: If the module cannot be imported. + AttributeError: If the function doesn't exist in the module. + TypeError: If the resolved object is not callable. + """ path = task.split('.') module = importlib.import_module('.'.join(path[:-1])) func = getattr(module, path[-1]) @@ -67,7 +122,15 @@ def _callable_task(task): return func def clean_task(self): - """Custom validation of the task field.""" + """ + Validates that the task field contains a valid callable path. + + Called automatically during model validation. Ensures the dotted path + can be imported and resolved to a callable. + + Raises: + ValidationError: If the task path is invalid or not callable. + """ try: self._callable_task(self.task) except (ImportError, AttributeError, TypeError, ValueError) as e: @@ -80,7 +143,15 @@ def clean_task(self): }) def clean_args(self): - """Custom validation of the args field.""" + """ + Validates that the args field contains valid JSON. + + Called automatically during model validation. Ensures the args field + can be parsed as JSON (should be a dict when deserialized). + + Raises: + ValidationError: If the args field is not valid JSON. + """ if self.args is None or self.args == "": return diff --git a/django_simple_queue/monitor.py b/django_simple_queue/monitor.py index f55217c..6aba0fa 100644 --- a/django_simple_queue/monitor.py +++ b/django_simple_queue/monitor.py @@ -1,3 +1,10 @@ +""" +Task monitoring and orphan detection utilities. + +This module provides functions for detecting and handling tasks whose worker +processes have died unexpectedly, as well as handling task timeouts and +subprocess failures. +""" import os from django.db import transaction @@ -7,7 +14,21 @@ def detect_orphaned_tasks(): - """Check for PROGRESS tasks with dead worker PIDs and mark them FAILED.""" + """ + Detect and mark orphaned tasks as failed. + + Scans all tasks with status PROGRESS and checks if their worker process + (identified by worker_pid) is still running. If the process is dead, + marks the task as FAILED and fires the on_failure signal. + + This function is called periodically by the task_worker command to clean + up tasks whose workers crashed unexpectedly. + + Note: + Uses ``select_for_update(skip_locked=True)`` to avoid blocking other workers. + If the PID exists but belongs to a different user (PermissionError), + the worker is assumed to still be alive. + """ with transaction.atomic(): in_progress = Task.objects.select_for_update(skip_locked=True).filter( status=Task.PROGRESS, worker_pid__isnull=False @@ -30,7 +51,16 @@ def detect_orphaned_tasks(): def handle_subprocess_exit(task_id, exit_code): - """Handle non-zero subprocess exit codes.""" + """ + Handle a task subprocess that exited with a non-zero code. + + Called by the task_worker after the subprocess finishes. If the exit code + indicates failure, marks the task as FAILED and fires the on_failure signal. + + Args: + task_id: UUID of the task that was being executed. + exit_code: The subprocess exit code (None or 0 means success). + """ if exit_code is None or exit_code == 0: return task = Task.objects.get(id=task_id) @@ -43,7 +73,17 @@ def handle_subprocess_exit(task_id, exit_code): def handle_task_timeout(task_id, timeout_seconds): - """Mark a task as failed due to exceeding the timeout.""" + """ + Mark a task as failed due to exceeding the timeout. + + Called by the task_worker when a subprocess doesn't complete within the + configured timeout. Marks the task as FAILED and fires the on_failure + signal with a TimeoutError. + + Args: + task_id: UUID of the task that timed out. + timeout_seconds: The timeout value that was exceeded. + """ task = Task.objects.get(id=task_id) if task.status == Task.PROGRESS: task.error = (task.error or "") + ( diff --git a/django_simple_queue/signals.py b/django_simple_queue/signals.py index 8dc1d2a..689b59b 100644 --- a/django_simple_queue/signals.py +++ b/django_simple_queue/signals.py @@ -1,7 +1,61 @@ +""" +Signals emitted during task lifecycle. + +This module defines Django signals that are fired at various points during +task execution, allowing you to hook into the task lifecycle. + +Signals: + before_job: Fired before task execution begins. + - sender: Task class + - task: The Task instance being executed + + on_success: Fired when a task completes successfully. + - sender: Task class + - task: The completed Task instance + + on_failure: Fired when a task fails with an exception or timeout. + - sender: Task class + - task: The failed Task instance + - error: The exception that caused the failure (may be None for orphaned tasks) + + before_loop: Fired before each iteration of a generator task. + - sender: Task class + - task: The Task instance + - iteration: Current iteration index (0-based) + + after_loop: Fired after each iteration of a generator task. + - sender: Task class + - task: The Task instance + - output: The value yielded by the generator + - iteration: Current iteration index (0-based) + +Example: + Connecting to signals:: + + from django.dispatch import receiver + from django_simple_queue.signals import on_success, on_failure + + @receiver(on_success) + def handle_task_success(sender, task, **kwargs): + print(f"Task {task.id} completed successfully!") + + @receiver(on_failure) + def handle_task_failure(sender, task, error, **kwargs): + print(f"Task {task.id} failed: {error}") +""" import django.dispatch -before_job = django.dispatch.Signal() # kwargs: task -on_success = django.dispatch.Signal() # kwargs: task -on_failure = django.dispatch.Signal() # kwargs: task, error -before_loop = django.dispatch.Signal() # kwargs: task, iteration -after_loop = django.dispatch.Signal() # kwargs: task, output, iteration +before_job = django.dispatch.Signal() +"""Signal fired before task execution begins. Provides: task.""" + +on_success = django.dispatch.Signal() +"""Signal fired when a task completes successfully. Provides: task.""" + +on_failure = django.dispatch.Signal() +"""Signal fired when a task fails. Provides: task, error.""" + +before_loop = django.dispatch.Signal() +"""Signal fired before each generator iteration. Provides: task, iteration.""" + +after_loop = django.dispatch.Signal() +"""Signal fired after each generator iteration. Provides: task, output, iteration.""" diff --git a/django_simple_queue/urls.py b/django_simple_queue/urls.py index baab836..1a9fde9 100644 --- a/django_simple_queue/urls.py +++ b/django_simple_queue/urls.py @@ -1,4 +1,4 @@ -from django.urls import re_path, path +from django.urls import path from django_simple_queue.views import ( view_task_status, ) diff --git a/django_simple_queue/views.py b/django_simple_queue/views.py index fe71e84..439b18d 100644 --- a/django_simple_queue/views.py +++ b/django_simple_queue/views.py @@ -1,3 +1,8 @@ +""" +Views for django-simple-queue. + +Provides HTTP endpoints for checking task status. +""" from django.core.exceptions import ValidationError from django.http import JsonResponse, HttpResponseBadRequest from django.shortcuts import render @@ -6,7 +11,22 @@ def view_task_status(request): - """View for displaying the status of the task.""" + """ + Display or return the status of a task. + + GET Parameters: + task_id (required): UUID of the task to query. + type (optional): Set to "json" for JSON response, otherwise renders HTML. + + Returns: + - JSON response with task data if type=json + - HTML page with task details otherwise + - HttpResponseBadRequest if task_id is missing, invalid, or not found + + Example: + GET /django_simple_queue/task?task_id=abc123 + GET /django_simple_queue/task?task_id=abc123&type=json + """ task_id = request.GET.get("task_id") if not task_id: return HttpResponseBadRequest("Missing task_id parameter.") diff --git a/django_simple_queue/worker.py b/django_simple_queue/worker.py index af7c82a..f7adc6f 100644 --- a/django_simple_queue/worker.py +++ b/django_simple_queue/worker.py @@ -1,3 +1,9 @@ +""" +Task execution worker module. + +This module contains the core logic for executing tasks in a subprocess, +including support for generator functions and log capture. +""" import asyncio import importlib import inspect @@ -12,6 +18,17 @@ class ManagedEventLoop: + """ + Context manager for asyncio event loop management. + + Ensures an event loop exists for the current context, creating one if + necessary. Cleans up the loop on exit. + + Example: + with ManagedEventLoop() as loop: + loop.run_until_complete(async_func()) + """ + def __init__(self): self.loop = None @@ -29,6 +46,27 @@ def __exit__(self, exc_type, exc_value, exc_tb): def execute_task(task_id, log_fd=None): + """ + Execute a task by its ID. + + This function is called in a subprocess by the task_worker command. + It handles loading the callable, executing it with the provided arguments, + and updating the task status/output in the database. + + For generator functions, each yielded value is appended to the output, + and before_loop/after_loop signals are fired for each iteration. + + Args: + task_id: UUID of the task to execute. + log_fd: Optional file descriptor for capturing stdout/stderr/logging. + If provided, all output is redirected to this descriptor. + + Signals Fired: + - before_job: Before execution starts + - on_success: If task completes successfully + - on_failure: If task raises an exception + - before_loop/after_loop: For each generator iteration + """ log_file = None log_handler = None if log_fd is not None: diff --git a/docs/advanced/admin.md b/docs/advanced/admin.md new file mode 100644 index 0000000..5062900 --- /dev/null +++ b/docs/advanced/admin.md @@ -0,0 +1,188 @@ +# Admin Interface + +Django Simple Queue includes a customized Django Admin interface for viewing and managing tasks. + +## Features + +The `TaskAdmin` class provides: + +- **Read-only fields**: All fields are read-only when editing existing tasks +- **Status page link**: Clickable link to the task status page +- **Enqueue action**: Re-queue failed or cancelled tasks +- **Search**: Find tasks by ID, callable path, or output +- **Filters**: Filter by status, created date, modified date + +## Accessing the Admin + +After adding `django_simple_queue` to `INSTALLED_APPS` and running migrations, access tasks at: + +``` +http://localhost:8000/admin/django_simple_queue/task/ +``` + +## List View + +The task list displays: + +| Column | Description | +|--------|-------------| +| ID | Task UUID (clickable for detail view) | +| Created | When the task was created | +| Modified | When the task was last updated | +| Task | Dotted path to the callable | +| Status | Clickable link to status page (opens in new tab) | + +### Default Ordering + +Tasks are ordered by `modified` descending (most recent first). + +## Detail View + +All fields are read-only. Displays: + +- All task metadata +- Full output, error, and log content +- Worker PID (if in progress) + +## Actions + +### Enqueue + +Select one or more tasks and choose "Enqueue" from the action dropdown to change their status to `QUEUED`. + +This is useful for: + +- Retrying failed tasks +- Re-running completed tasks +- Processing cancelled tasks + +!!! warning "No Cleanup" + The Enqueue action only changes status. It does not clear `error`, `output`, or `log` fields. Consider clearing these manually or via code if needed. + +## Filtering + +### By Status + +Filter tasks by their current status: + +- Queued +- In progress +- Completed +- Failed +- Cancelled + +### By Date + +Filter by: + +- Created date +- Modified date + +Options include: Today, Past 7 days, This month, This year. + +## Searching + +Search across: + +- Task ID (UUID) +- Task path (callable) +- Output content + +Example searches: + +- `send_email` - Find all email tasks +- `failed` - Search in output/error text +- `abc123` - Find task by partial ID + +## Customization + +### Extending TaskAdmin + +```python +# myapp/admin.py +from django.contrib import admin +from django_simple_queue.admin import TaskAdmin +from django_simple_queue.models import Task + +# Unregister the default admin +admin.site.unregister(Task) + +# Register your customized version +@admin.register(Task) +class CustomTaskAdmin(TaskAdmin): + list_display = ('id', 'task', 'status_page_link', 'created', 'duration') + + def duration(self, obj): + if obj.status in (Task.COMPLETED, Task.FAILED): + delta = obj.modified - obj.created + return f"{delta.total_seconds():.1f}s" + return "-" + duration.short_description = "Duration" +``` + +### Adding Custom Actions + +```python +@admin.register(Task) +class CustomTaskAdmin(TaskAdmin): + + @admin.action(description='Mark as cancelled') + def cancel_tasks(self, request, queryset): + updated = queryset.filter( + status__in=[Task.QUEUED, Task.PROGRESS] + ).update(status=Task.CANCELLED) + self.message_user(request, f"{updated} tasks cancelled") + + actions = TaskAdmin.actions + ['cancel_tasks'] +``` + +### Custom Filters + +```python +from django.contrib import admin +from django.utils import timezone +from datetime import timedelta + +class SlowTaskFilter(admin.SimpleListFilter): + title = 'Performance' + parameter_name = 'slow' + + def lookups(self, request, model_admin): + return [ + ('slow', 'Slow (>1 min)'), + ('fast', 'Fast (<1 min)'), + ] + + def queryset(self, request, queryset): + if self.value() == 'slow': + return queryset.filter( + status__in=[Task.COMPLETED, Task.FAILED], + ).extra( + where=["modified - created > interval '1 minute'"] + ) + # ... etc + +@admin.register(Task) +class CustomTaskAdmin(TaskAdmin): + list_filter = TaskAdmin.list_filter + (SlowTaskFilter,) +``` + +## API Reference + +::: django_simple_queue.admin.TaskAdmin + options: + show_source: true + members: + - get_readonly_fields + - status_page_link + - enqueue_tasks + +## Security Considerations + +The admin interface allows: + +- Viewing task arguments (may contain sensitive data) +- Viewing task output and errors +- Re-queuing tasks (which will execute again) + +Ensure admin access is properly restricted in production. diff --git a/docs/advanced/databases.md b/docs/advanced/databases.md new file mode 100644 index 0000000..ab1ec3d --- /dev/null +++ b/docs/advanced/databases.md @@ -0,0 +1,239 @@ +# Database Backends + +Django Simple Queue uses your existing database as the task queue broker. This page covers database-specific considerations. + +## Recommended: PostgreSQL + +PostgreSQL is the recommended database for production use due to its robust locking features. + +### Key Features + +- **`SELECT FOR UPDATE SKIP LOCKED`**: Allows multiple workers to claim tasks concurrently without blocking each other +- **ACID transactions**: Ensures task state changes are atomic +- **Good performance**: Efficient row-level locking + +### Configuration + +```python +# settings.py +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'mydb', + 'USER': 'myuser', + 'PASSWORD': 'mypassword', + 'HOST': 'localhost', + 'PORT': '5432', + 'CONN_MAX_AGE': 600, # Keep connections open for 10 minutes + } +} +``` + +### Connection Pool Settings + +For production with multiple workers: + +```python +# settings.py +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + # ... + 'CONN_MAX_AGE': None, # Persistent connections + 'CONN_HEALTH_CHECKS': True, # Django 4.1+ + 'OPTIONS': { + 'connect_timeout': 10, + }, + } +} +``` + +## SQLite + +SQLite is suitable for development and single-worker setups. + +### Limitations + +- **No `SKIP LOCKED`**: Falls back to basic `SELECT FOR UPDATE` +- **Database locking**: May experience contention with multiple workers +- **File-based**: Not suitable for distributed deployments + +### When to Use + +- Local development +- Testing +- Single worker deployments +- Low-volume applications + +### Configuration + +```python +# settings.py (development) +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} +``` + +## MySQL/MariaDB + +MySQL 8.0+ and MariaDB 10.3+ support `SKIP LOCKED`. + +### Configuration + +```python +# settings.py +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'mydb', + 'USER': 'myuser', + 'PASSWORD': 'mypassword', + 'HOST': 'localhost', + 'PORT': '3306', + 'OPTIONS': { + 'charset': 'utf8mb4', + }, + } +} +``` + +### Considerations + +- Ensure InnoDB engine for row-level locking +- Use READ COMMITTED isolation level for best concurrency + +## How Task Claiming Works + +The worker uses database-level pessimistic locking: + +```python +# Simplified from task_worker.py +with transaction.atomic(): + try: + # Try to use SKIP LOCKED for better concurrency + qs = Task.objects.select_for_update(skip_locked=True) + except NotSupportedError: + # Fallback for databases without skip_locked + qs = Task.objects.select_for_update() + + # Get the oldest queued task + task = qs.filter(status=Task.QUEUED).order_by('modified').first() + + if task: + # Claim the task within the transaction + task.status = Task.PROGRESS + task.worker_pid = os.getpid() + task.save() +``` + +### With `SKIP LOCKED` (PostgreSQL, MySQL 8+) + +``` +Worker A: SELECT ... WHERE status=QUEUED FOR UPDATE SKIP LOCKED → Gets Task 1 +Worker B: SELECT ... WHERE status=QUEUED FOR UPDATE SKIP LOCKED → Gets Task 2 (skips locked Task 1) +``` + +Both workers proceed immediately without waiting. + +### Without `SKIP LOCKED` (SQLite) + +``` +Worker A: SELECT ... WHERE status=QUEUED FOR UPDATE → Gets Task 1 +Worker B: SELECT ... WHERE status=QUEUED FOR UPDATE → Waits for Worker A to commit +Worker A: Commits → Task 1 now PROGRESS +Worker B: Gets Task 2 +``` + +Worker B must wait, reducing concurrency. + +## Multiple Workers + +### With PostgreSQL + +Run as many workers as needed: + +```bash +# Start 4 workers +for i in {1..4}; do + python manage.py task_worker & +done +``` + +Each worker claims different tasks thanks to `SKIP LOCKED`. + +### With SQLite + +Limit to 1-2 workers to avoid contention: + +```bash +# Single worker recommended for SQLite +python manage.py task_worker +``` + +## Indexing + +The Task model should have indexes on commonly queried fields. The default migration includes: + +- Primary key on `id` (UUID) +- Index on `status` (for filtering queued tasks) +- Index on `modified` (for ordering) + +For high-volume queues, consider: + +```python +# Custom migration for additional indexes +from django.db import migrations, models + +class Migration(migrations.Migration): + dependencies = [ + ('django_simple_queue', 'XXXX_previous'), + ] + + operations = [ + migrations.AddIndex( + model_name='task', + index=models.Index( + fields=['status', 'modified'], + name='status_modified_idx' + ), + ), + ] +``` + +## Cleanup Old Tasks + +Regularly clean up completed/failed tasks to maintain performance: + +```python +from django.core.management.base import BaseCommand +from django.utils import timezone +from datetime import timedelta +from django_simple_queue.models import Task + +class Command(BaseCommand): + help = 'Clean up old completed tasks' + + def add_arguments(self, parser): + parser.add_argument('--days', type=int, default=30) + + def handle(self, *args, **options): + cutoff = timezone.now() - timedelta(days=options['days']) + deleted, _ = Task.objects.filter( + status__in=[Task.COMPLETED, Task.FAILED], + modified__lt=cutoff + ).delete() + self.stdout.write(f"Deleted {deleted} tasks") +``` + +## Database Comparison + +| Feature | PostgreSQL | MySQL 8+ | SQLite | +|---------|------------|----------|--------| +| `SKIP LOCKED` | Yes | Yes | No | +| Multiple workers | Excellent | Good | Limited | +| Row-level locking | Yes | Yes (InnoDB) | No | +| Production ready | Yes | Yes | Dev only | +| Distributed setup | Yes | Yes | No | diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md new file mode 100644 index 0000000..28fdc60 --- /dev/null +++ b/docs/getting-started/configuration.md @@ -0,0 +1,111 @@ +# Configuration + +All settings are configured in your Django `settings.py` with the `DJANGO_SIMPLE_QUEUE_` prefix. + +## Available Settings + +### DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS + +**Type:** `set[str] | None` +**Default:** `None` (all tasks allowed) + +Restricts which callables can be executed. When set, only tasks in this set can be created. + +```python +DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = { + "myapp.tasks.send_email", + "myapp.tasks.process_order", + "myapp.tasks.generate_report", +} +``` + +!!! warning "Security Recommendation" + Always configure an allowlist in production. Without it, any callable in your codebase could potentially be executed. + +Set to an empty set to disallow all tasks: + +```python +DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = set() # No tasks allowed +``` + +--- + +### DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT + +**Type:** `int | None` +**Default:** `3600` (1 hour) + +Maximum execution time for a task in seconds. Tasks exceeding this timeout are terminated and marked as failed. + +```python +# 5 minute timeout +DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = 300 + +# 30 minute timeout +DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = 1800 + +# Disable timeout (tasks can run indefinitely) +DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = None +``` + +!!! tip + Set appropriate timeouts based on your longest-running tasks. Tasks that time out will be terminated with SIGTERM, then SIGKILL if they don't stop. + +--- + +### DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE + +**Type:** `int` +**Default:** `10485760` (10 MB) + +Maximum size in bytes for task output stored in the database. + +```python +# 1 MB limit +DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE = 1_000_000 + +# 100 KB limit +DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE = 100_000 +``` + +--- + +### DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE + +**Type:** `int` +**Default:** `1048576` (1 MB) + +Maximum size in bytes for the JSON-serialized task arguments. + +```python +# 100 KB limit +DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE = 100_000 +``` + +## Example Configuration + +```python +# settings.py + +# Only allow specific tasks to be executed +DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = { + "orders.tasks.process_order", + "orders.tasks.send_confirmation", + "reports.tasks.generate_daily_report", + "reports.tasks.send_report_email", +} + +# Tasks timeout after 10 minutes +DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = 600 + +# Limit output size to 5 MB +DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE = 5 * 1024 * 1024 + +# Limit args size to 500 KB +DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE = 500 * 1024 +``` + +## Next Steps + +- See the [Quick Start](quickstart.md) for a complete example +- Learn about [worker optimization](../guides/worker-optimization.md) for production deployments diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..ee5d6ff --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,98 @@ +# Installation + +## Requirements + +- Python 3.8+ +- Django 3.2+ + +## Install the Package + +```bash +pip install django-simple-queue +``` + +## Configure Django + +### 1. Add to INSTALLED_APPS + +Add `django_simple_queue` to your `INSTALLED_APPS` in `settings.py`: + +```python +INSTALLED_APPS = [ + # Django apps... + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + + # Third-party apps... + 'django_simple_queue', + + # Your apps... + 'myapp', +] +``` + +### 2. Add URL Configuration + +Include the task status URL in your `urls.py`: + +```python +from django.urls import path, include + +urlpatterns = [ + # ... + path('django_simple_queue/', include('django_simple_queue.urls')), +] +``` + +### 3. Run Migrations + +Apply the database migrations to create the Task table: + +```bash +python manage.py migrate django_simple_queue +``` + +## Start the Worker + +Run the worker command to start processing tasks: + +```bash +python manage.py task_worker +``` + +!!! tip "Running in Production" + In production, use a process manager like systemd, supervisor, or Docker to keep the worker running. You can run multiple workers for parallel processing. + +## Verify Installation + +1. Create a simple task function: + + ```python + # myapp/tasks.py + def hello_world(name): + return f"Hello, {name}!" + ``` + +2. Enqueue a task from Django shell: + + ```python + from django_simple_queue.utils import create_task + + task_id = create_task( + task="myapp.tasks.hello_world", + args={"name": "World"} + ) + print(f"Created task: {task_id}") + ``` + +3. Check the task status at `/django_simple_queue/task?task_id=` or in the Django admin. + +## Next Steps + +- [Configure settings](configuration.md) for task timeouts, allowed tasks, etc. +- Learn how to [create tasks](../guides/creating-tasks.md) in your application +- Understand the [task lifecycle](../guides/lifecycle.md) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..847cc97 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,179 @@ +# Quick Start + +This guide walks you through creating and running your first background task. + +## 1. Create a Task Function + +Create a module for your task functions. The function should accept keyword arguments and return a string (or yield strings for generators). + +```python +# myapp/tasks.py + +def process_order(order_id, notify_customer=True): + """ + Process an order in the background. + + Args: + order_id: The ID of the order to process + notify_customer: Whether to send notification email + + Returns: + A status message + """ + from myapp.models import Order + + order = Order.objects.get(id=order_id) + + # Do the processing... + order.status = 'processed' + order.save() + + if notify_customer: + # Send email... + pass + + return f"Order {order_id} processed successfully" +``` + +## 2. Configure Allowed Tasks + +Add your task to the allowlist in `settings.py`: + +```python +DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = { + "myapp.tasks.process_order", +} +``` + +## 3. Create a Task + +Use the `create_task` utility to enqueue tasks: + +```python +# In a view, signal handler, or anywhere in your code +from django_simple_queue.utils import create_task + +def place_order(request): + # Create the order... + order = Order.objects.create(...) + + # Queue background processing + task_id = create_task( + task="myapp.tasks.process_order", + args={ + "order_id": order.id, + "notify_customer": True + } + ) + + return JsonResponse({ + "order_id": order.id, + "task_id": str(task_id) + }) +``` + +## 4. Start the Worker + +Run the worker command in a terminal: + +```bash +python manage.py task_worker +``` + +You'll see output like: + +``` +Task timeout configured: 3600 seconds +2024-01-15 10:30:00: [RAM Usage: 45.2 MB] Heartbeat.. +Initiating task id: abc123-def456... +Finished task id: abc123-def456 +``` + +## 5. Check Task Status + +### Via URL + +Visit `/django_simple_queue/task?task_id=` for an HTML status page, or add `&type=json` for JSON response: + +```bash +curl "http://localhost:8000/django_simple_queue/task?task_id=abc123&type=json" +``` + +```json +{ + "id": "abc123-def456-...", + "created": "2024-01-15 10:30:00", + "status": "Completed", + "output": "Order 42 processed successfully", + "error": null +} +``` + +### Via Code + +```python +from django_simple_queue.models import Task + +task = Task.objects.get(id=task_id) +print(task.status) # 2 (Task.COMPLETED) +print(task.get_status_display()) # "Completed" +print(task.output) # "Order 42 processed successfully" +``` + +### Via Admin + +Navigate to Django Admin at `/admin/django_simple_queue/task/` to view all tasks with filtering and search. + +## Complete Example + +Here's a full example showing a view that creates a task and returns immediately: + +```python +# myapp/views.py +from django.http import JsonResponse +from django.views.decorators.http import require_POST +from django_simple_queue.utils import create_task + +@require_POST +def start_report_generation(request): + """Start generating a report in the background.""" + report_type = request.POST.get('report_type', 'daily') + + task_id = create_task( + task="myapp.tasks.generate_report", + args={ + "report_type": report_type, + "user_id": request.user.id + } + ) + + return JsonResponse({ + "message": "Report generation started", + "task_id": str(task_id), + "status_url": f"/django_simple_queue/task?task_id={task_id}&type=json" + }) + + +def check_task_status(request, task_id): + """Check the status of a background task.""" + from django_simple_queue.models import Task + + try: + task = Task.objects.get(id=task_id) + except Task.DoesNotExist: + return JsonResponse({"error": "Task not found"}, status=404) + + return JsonResponse({ + "id": str(task.id), + "status": task.get_status_display(), + "output": task.output, + "error": task.error + }) +``` + +## Next Steps + +- Learn about [task lifecycle](../guides/lifecycle.md) and status transitions +- Use [signals](../guides/signals.md) to react to task events +- Handle [errors](../guides/errors.md) gracefully +- Optimize for production with [worker optimization](../guides/worker-optimization.md) diff --git a/docs/guides/creating-tasks.md b/docs/guides/creating-tasks.md new file mode 100644 index 0000000..445c0c5 --- /dev/null +++ b/docs/guides/creating-tasks.md @@ -0,0 +1,240 @@ +# Creating Tasks + +This guide covers how to define task functions and enqueue them for execution. + +## Defining Task Functions + +Task functions are regular Python functions that: + +1. Accept **keyword arguments only** (passed as a dict) +2. Return a **string** (stored in task output) +3. Are importable via a dotted path + +```python +# myapp/tasks.py + +def send_email(to, subject, body, cc=None): + """Send an email in the background.""" + import smtplib + # Send email logic... + return f"Email sent to {to}" + + +def process_image(image_id, resize_to=None, format="png"): + """Process and optionally resize an image.""" + from myapp.models import Image + image = Image.objects.get(id=image_id) + # Processing logic... + return f"Image {image_id} processed" +``` + +## Enqueuing Tasks + +Use the `create_task` function to add tasks to the queue: + +```python +from django_simple_queue.utils import create_task + +# Basic usage +task_id = create_task( + task="myapp.tasks.send_email", + args={ + "to": "user@example.com", + "subject": "Hello", + "body": "Welcome to our service!" + } +) + +# With optional arguments +task_id = create_task( + task="myapp.tasks.process_image", + args={ + "image_id": 42, + "resize_to": (800, 600), + "format": "webp" + } +) +``` + +## Arguments Format + +The `args` parameter must be a **dictionary** that is JSON-serializable: + +```python +# Valid argument types +args = { + "string": "hello", + "number": 42, + "float": 3.14, + "boolean": True, + "null": None, + "list": [1, 2, 3], + "nested": {"key": "value"} +} + +# NOT valid - will raise TypeError +args = ["positional", "args"] # Must be dict, not list +``` + +!!! warning "No Positional Arguments" + Task functions receive arguments as `**kwargs`, so all arguments must be keyword arguments in the dict. + +## Return Values + +Task functions should return a string. The return value is stored in `task.output`: + +```python +def example_task(data): + result = process(data) + return f"Processed {len(result)} items" # Stored in task.output +``` + +For complex return data, serialize to JSON: + +```python +import json + +def example_task(data): + result = {"processed": 100, "failed": 2} + return json.dumps(result) +``` + +## Best Practices + +### 1. Import Inside Functions + +Import dependencies inside the function to avoid loading them at worker startup: + +```python +# Good - imports only when task runs +def send_notification(user_id, message): + from myapp.models import User + from myapp.services import NotificationService + + user = User.objects.get(id=user_id) + NotificationService.send(user, message) + return "Notification sent" + + +# Avoid - loads models at import time +from myapp.models import User # Loaded when worker starts + +def send_notification(user_id, message): + user = User.objects.get(id=user_id) + ... +``` + +### 2. Pass IDs, Not Objects + +Pass database IDs instead of model instances: + +```python +# Good - passes ID +task_id = create_task( + task="myapp.tasks.process_order", + args={"order_id": order.id} +) + +# Bad - can't serialize model instance +task_id = create_task( + task="myapp.tasks.process_order", + args={"order": order} # TypeError! +) +``` + +### 3. Keep Tasks Focused + +Each task should do one thing: + +```python +# Good - separate tasks +def send_welcome_email(user_id): ... +def create_default_settings(user_id): ... +def notify_admin(user_id): ... + +# Then create multiple tasks +for task_func in ["send_welcome_email", "create_default_settings", "notify_admin"]: + create_task(task=f"myapp.tasks.{task_func}", args={"user_id": user.id}) +``` + +### 4. Handle Failures Gracefully + +Tasks should handle expected errors and provide useful error messages: + +```python +def process_payment(payment_id): + from myapp.models import Payment + from myapp.exceptions import PaymentError + + try: + payment = Payment.objects.get(id=payment_id) + except Payment.DoesNotExist: + return f"Payment {payment_id} not found" + + try: + result = payment.process() + return f"Payment {payment_id} processed: {result}" + except PaymentError as e: + # Log for debugging, return error message + logger.exception("Payment processing failed") + raise # Re-raise to mark task as failed +``` + +## Creating Tasks from Various Contexts + +### From Views + +```python +from django.http import JsonResponse +from django_simple_queue.utils import create_task + +def upload_file(request): + file = request.FILES['file'] + saved_path = save_file(file) + + task_id = create_task( + task="myapp.tasks.process_upload", + args={"file_path": saved_path} + ) + + return JsonResponse({"task_id": str(task_id)}) +``` + +### From Signals + +```python +from django.db.models.signals import post_save +from django.dispatch import receiver +from myapp.models import Order +from django_simple_queue.utils import create_task + +@receiver(post_save, sender=Order) +def order_created(sender, instance, created, **kwargs): + if created: + create_task( + task="myapp.tasks.process_new_order", + args={"order_id": instance.id} + ) +``` + +### From Management Commands + +```python +from django.core.management.base import BaseCommand +from django_simple_queue.utils import create_task + +class Command(BaseCommand): + def handle(self, *args, **options): + for user_id in User.objects.values_list('id', flat=True): + create_task( + task="myapp.tasks.send_newsletter", + args={"user_id": user_id} + ) + self.stdout.write("Newsletter tasks created") +``` + +## Next Steps + +- Understand the [task lifecycle](lifecycle.md) +- Use [generator functions](generators.md) for streaming output +- Handle [errors](errors.md) and failures diff --git a/docs/guides/errors.md b/docs/guides/errors.md new file mode 100644 index 0000000..a9ed020 --- /dev/null +++ b/docs/guides/errors.md @@ -0,0 +1,325 @@ +# Error Handling + +This guide covers how errors are captured, stored, and handled in Django Simple Queue. + +## Error Storage + +When a task fails, error information is stored in separate fields: + +| Field | Contents | +|-------|----------| +| `output` | Return value from the task function (may be partial for generators) | +| `error` | Exception representation and traceback | +| `log` | Captured stdout, stderr, and Python logging output | + +## Types of Failures + +### 1. Exception in Task Function + +When your task raises an exception: + +```python +def failing_task(x): + if x < 0: + raise ValueError("x must be non-negative") + return f"Processed {x}" +``` + +The `error` field contains: + +``` +ValueError('x must be non-negative') + +Traceback (most recent call last): + File ".../worker.py", line 76, in execute_task + task_obj.output = func(**args) + File ".../myapp/tasks.py", line 3, in failing_task + raise ValueError("x must be non-negative") +ValueError: x must be non-negative +``` + +### 2. Timeout + +When a task exceeds `DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT`: + +``` +Task timed out after 300 seconds +``` + +### 3. Worker Process Crash + +When the worker subprocess exits unexpectedly: + +``` +Worker subprocess exited with code 1 +``` + +### 4. Orphaned Task + +When the worker process dies while task is in progress: + +``` +Task failed: worker process (PID 12345) no longer running +``` + +## Reading Error Information + +```python +from django_simple_queue.models import Task + +task = Task.objects.get(id=task_id) + +if task.status == Task.FAILED: + print("Task failed!") + print(f"Error: {task.error}") + print(f"Logs: {task.log}") + print(f"Partial output: {task.output}") +``` + +## Handling Errors in Task Functions + +### Catching Expected Errors + +```python +def resilient_task(item_id): + from myapp.models import Item + + try: + item = Item.objects.get(id=item_id) + except Item.DoesNotExist: + return f"Item {item_id} not found" # Return error message, don't fail + + try: + result = process_item(item) + return f"Processed: {result}" + except ProcessingError as e: + # Re-raise to mark task as failed + raise +``` + +### Partial Success in Generators + +For generator tasks, you can yield error messages while continuing: + +```python +def batch_process(item_ids): + succeeded = 0 + failed = 0 + + for item_id in item_ids: + try: + process_item(item_id) + succeeded += 1 + yield f"Processed {item_id}\n" + except Exception as e: + failed += 1 + yield f"ERROR processing {item_id}: {e}\n" + + yield f"Complete: {succeeded} succeeded, {failed} failed" +``` + +## Using Signals for Error Handling + +### Logging Failures + +```python +from django.dispatch import receiver +from django_simple_queue.signals import on_failure +import logging + +logger = logging.getLogger('tasks') + +@receiver(on_failure) +def log_task_failure(sender, task, error, **kwargs): + logger.error( + f"Task {task.id} ({task.task}) failed", + extra={ + 'task_id': str(task.id), + 'task_path': task.task, + 'error': str(error) if error else task.error, + } + ) +``` + +### Alerting on Critical Failures + +```python +@receiver(on_failure) +def alert_on_critical_failure(sender, task, error, **kwargs): + critical_tasks = ['payments.tasks.', 'orders.tasks.'] + + if any(task.task.startswith(prefix) for prefix in critical_tasks): + send_alert( + channel='#alerts', + message=f"Critical task failed: {task.task}\nError: {error}" + ) +``` + +### Automatic Retry + +```python +from django_simple_queue.utils import create_task +import json + +MAX_RETRIES = 3 + +@receiver(on_failure) +def retry_failed_task(sender, task, error, **kwargs): + args = json.loads(task.args) if task.args else {} + retry_count = args.get('_retry_count', 0) + + # Only retry certain tasks + retryable = ['myapp.tasks.send_email', 'myapp.tasks.sync_data'] + if task.task not in retryable: + return + + if retry_count < MAX_RETRIES: + create_task( + task=task.task, + args={**args, '_retry_count': retry_count + 1} + ) + logger.info(f"Scheduled retry {retry_count + 1} for task {task.id}") +``` + +## Re-queuing Failed Tasks + +### Via Code + +```python +def requeue_failed_task(task_id): + task = Task.objects.get(id=task_id) + if task.status == Task.FAILED: + task.status = Task.QUEUED + task.error = None + task.output = None + task.log = None + task.worker_pid = None + task.save() +``` + +### Via Admin + +Use the "Enqueue" action in Django Admin to re-queue selected tasks. + +### Bulk Re-queue + +```python +def requeue_all_failed(): + return Task.objects.filter(status=Task.FAILED).update( + status=Task.QUEUED, + error=None, + output=None, + log=None, + worker_pid=None + ) +``` + +## Debugging Failed Tasks + +### View Full Error + +```python +task = Task.objects.get(id=task_id) +print(task.error) # Full traceback +``` + +### Check Captured Logs + +```python +print(task.log) # stdout, stderr, logging output +``` + +### Reproduce Locally + +```python +# Get the args that were passed +import json +args = json.loads(task.args) + +# Import and call the function directly +from myapp.tasks import my_task +my_task(**args) # Will raise the exception +``` + +## Best Practices + +### 1. Use Logging + +Configure logging in your task functions: + +```python +import logging + +logger = logging.getLogger(__name__) + +def my_task(item_id): + logger.info(f"Starting task for item {item_id}") + try: + result = process(item_id) + logger.info(f"Completed: {result}") + return result + except Exception: + logger.exception("Task failed") + raise +``` + +All logging output is captured in `task.log`. + +### 2. Validate Early + +Check arguments at the start of the task: + +```python +def my_task(item_id, action): + if action not in ('create', 'update', 'delete'): + raise ValueError(f"Invalid action: {action}") + + if not Item.objects.filter(id=item_id).exists(): + raise ValueError(f"Item {item_id} not found") + + # Now proceed with confidence + ... +``` + +### 3. Clean Up on Failure + +Use try/finally for cleanup: + +```python +def process_file(file_path): + temp_file = None + try: + temp_file = create_temp_copy(file_path) + result = process(temp_file) + return result + finally: + if temp_file: + os.unlink(temp_file) +``` + +### 4. Don't Swallow Errors + +Let exceptions propagate so they're recorded: + +```python +# Bad - error is hidden +def my_task(): + try: + do_work() + except Exception: + pass # Task appears to succeed! + +# Good - error is recorded +def my_task(): + try: + do_work() + except TemporaryError: + raise # Will be marked as failed + except PermanentError as e: + return f"Permanent error: {e}" # Return error message +``` + +## Next Steps + +- [Worker optimization](worker-optimization.md) for production deployments +- [Task lifecycle](lifecycle.md) for understanding status transitions diff --git a/docs/guides/generators.md b/docs/guides/generators.md new file mode 100644 index 0000000..866bc27 --- /dev/null +++ b/docs/guides/generators.md @@ -0,0 +1,238 @@ +# Generator Functions + +Django Simple Queue supports generator functions for tasks that produce output incrementally. This is useful for long-running tasks where you want to track progress or stream results. + +## Basic Usage + +Define your task as a generator function using `yield`: + +```python +# myapp/tasks.py + +def process_items(items): + """Process items one at a time, yielding progress.""" + for i, item in enumerate(items): + # Do some work + result = process_single_item(item) + + # Yield progress update + yield f"Processed item {i+1}/{len(items)}: {result}\n" + + yield "All items processed!" +``` + +## How It Works + +When the worker detects a generator function: + +1. It iterates through the generator +2. Each yielded value is **appended** to `task.output` +3. The task is saved after each yield (visible in real-time) +4. `before_loop` and `after_loop` signals fire for each iteration + +``` +Generator yields: "Step 1 done\n" → "Step 2 done\n" → "Finished!" + +task.output: "Step 1 done\n" + "Step 1 done\nStep 2 done\n" + "Step 1 done\nStep 2 done\nFinished!" +``` + +## Progress Tracking Example + +```python +def import_csv(file_path, batch_size=100): + """Import CSV file with progress updates.""" + import csv + + with open(file_path) as f: + reader = list(csv.DictReader(f)) + total = len(reader) + + for i in range(0, total, batch_size): + batch = reader[i:i + batch_size] + + # Process batch + for row in batch: + process_row(row) + + progress = min(i + batch_size, total) + yield f"Imported {progress}/{total} rows ({progress*100//total}%)\n" + + yield f"Import complete: {total} rows processed" +``` + +## Monitoring Progress + +### Check Progress via API + +Poll the task status endpoint to see real-time output: + +```python +import requests +import time + +def poll_task_progress(task_id): + while True: + response = requests.get( + f"http://localhost:8000/django_simple_queue/task", + params={"task_id": task_id, "type": "json"} + ) + data = response.json() + + print(data["output"]) + + if data["status"] in ("Completed", "Failed"): + break + + time.sleep(1) +``` + +### Using Signals + +React to each iteration with signals: + +```python +from django.dispatch import receiver +from django_simple_queue.signals import after_loop + +@receiver(after_loop) +def on_progress(sender, task, output, iteration, **kwargs): + """Called after each yield.""" + print(f"Task {task.id} iteration {iteration}: {output}") + + # Example: Update a progress bar, send websocket message, etc. + if "myapp.tasks.import_csv" in task.task: + broadcast_progress(task.id, output) +``` + +## Use Cases + +### Large Data Processing + +```python +def process_large_dataset(dataset_id): + """Process records in chunks with progress.""" + from myapp.models import Record + + records = Record.objects.filter(dataset_id=dataset_id) + total = records.count() + processed = 0 + + for record in records.iterator(chunk_size=1000): + process_record(record) + processed += 1 + + if processed % 1000 == 0: + yield f"Processed {processed}/{total} records\n" + + yield f"Complete: processed {total} records" +``` + +### Multi-Step Pipeline + +```python +def run_pipeline(job_id): + """Run a multi-step pipeline with status updates.""" + yield "Step 1: Fetching data...\n" + data = fetch_data(job_id) + + yield "Step 2: Validating...\n" + validated = validate_data(data) + + yield "Step 3: Transforming...\n" + transformed = transform_data(validated) + + yield "Step 4: Saving results...\n" + save_results(job_id, transformed) + + yield f"Pipeline complete: processed {len(transformed)} items" +``` + +### File Download with Progress + +```python +def download_large_file(url, destination): + """Download file with progress reporting.""" + import requests + + response = requests.get(url, stream=True) + total_size = int(response.headers.get('content-length', 0)) + + downloaded = 0 + chunk_size = 8192 + + with open(destination, 'wb') as f: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + downloaded += len(chunk) + + if total_size: + progress = downloaded * 100 // total_size + yield f"Downloaded {downloaded}/{total_size} bytes ({progress}%)\n" + + yield f"Download complete: {destination}" +``` + +## Best Practices + +### 1. Yield Meaningful Updates + +```python +# Good - clear progress indication +yield f"Processing batch {batch_num}/{total_batches}\n" +yield f"Imported {count} records in {duration:.1f}s\n" + +# Less useful - too verbose +yield f"Processing record {i}\n" # Don't yield for every single item +``` + +### 2. Include Newlines + +Add newlines to output for readability: + +```python +yield "Step 1 complete\n" # Good - newline included +yield "Step 2 complete\n" +``` + +### 3. Final Summary + +End with a summary message: + +```python +def my_task(): + yield "Working...\n" + yield "Still working...\n" + yield "Done! Processed 1000 items in 45 seconds." # Summary +``` + +### 4. Handle Errors in Generator + +```python +def safe_generator_task(items): + for i, item in enumerate(items): + try: + result = process(item) + yield f"Item {i}: {result}\n" + except Exception as e: + yield f"Item {i}: ERROR - {e}\n" + # Continue processing or re-raise based on requirements + + yield "Processing complete" +``` + +## Comparison: Generator vs Regular Function + +| Aspect | Regular Function | Generator Function | +|--------|------------------|-------------------| +| Output | Single return value | Accumulated yields | +| Progress | Not visible until complete | Real-time updates | +| Signals | `before_job`, `on_success/failure` | + `before_loop`, `after_loop` | +| Memory | May need to hold all data | Can process incrementally | +| Use case | Quick tasks, single result | Long tasks, progress tracking | + +## Next Steps + +- Handle [errors](errors.md) in generator tasks +- Use [signals](signals.md) to react to `before_loop` and `after_loop` diff --git a/docs/guides/lifecycle.md b/docs/guides/lifecycle.md new file mode 100644 index 0000000..e15beb0 --- /dev/null +++ b/docs/guides/lifecycle.md @@ -0,0 +1,202 @@ +# Task Lifecycle + +Understanding how tasks move through different states helps you monitor execution and handle edge cases. + +## Task States + +Tasks progress through the following states: + +| Status | Value | Description | +|--------|-------|-------------| +| `QUEUED` | 0 | Task is waiting to be picked up by a worker | +| `PROGRESS` | 1 | Task is currently being executed | +| `COMPLETED` | 2 | Task finished successfully | +| `FAILED` | 3 | Task encountered an error | +| `CANCELLED` | 4 | Task was manually cancelled | + +```python +from django_simple_queue.models import Task + +# Check status by value +if task.status == Task.COMPLETED: + print("Task finished!") + +# Get human-readable status +print(task.get_status_display()) # "Completed" +``` + +## State Transitions + +``` + ┌──────────────┐ + │ QUEUED │ + │ (0) │ + └──────┬───────┘ + │ + Worker claims task + │ + ▼ + ┌──────────────┐ + ┌─────│ PROGRESS │─────┐ + │ │ (1) │ │ + │ └──────────────┘ │ + │ │ + Success Exception + or timeout or crash + │ │ + ▼ ▼ + ┌──────────────┐ ┌──────────────┐ + │ COMPLETED │ │ FAILED │ + │ (2) │ │ (3) │ + └──────────────┘ └──────────────┘ + + + ┌──────────────┐ + │ CANCELLED │ ← Set manually via admin or code + │ (4) │ + └──────────────┘ +``` + +## Worker Process Flow + +1. **Polling**: Worker polls for QUEUED tasks every 3-9 seconds (randomized) +2. **Claiming**: Uses `SELECT FOR UPDATE SKIP LOCKED` to claim exactly one task +3. **Status Update**: Sets status to PROGRESS and records `worker_pid` +4. **Subprocess**: Spawns a subprocess to execute the task function +5. **Completion**: Updates status to COMPLETED or FAILED based on result +6. **Cleanup**: Clears `worker_pid`, stores `log` output + +### Task Fields Updated During Execution + +| Field | When Updated | Description | +|-------|--------------|-------------| +| `status` | Claim, completion | Current execution state | +| `worker_pid` | Claim, completion | PID of worker (cleared when done) | +| `output` | During execution | Return value from task function | +| `error` | On failure | Exception message and traceback | +| `log` | After completion | Captured stdout/stderr/logging | +| `modified` | Any update | Last modification timestamp | + +## Failure Modes + +Tasks can fail in several ways: + +### 1. Exception in Task Function + +If the task function raises an exception: + +```python +def failing_task(x): + raise ValueError("Something went wrong") +``` + +- Status set to `FAILED` +- Exception and traceback stored in `error` field +- `on_failure` signal fired with the exception + +### 2. Timeout + +If task exceeds `DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT`: + +- Worker sends SIGTERM to subprocess +- After 5 seconds, sends SIGKILL if still alive +- Status set to `FAILED` +- Timeout message added to `error` field + +### 3. Worker Crash (Orphaned Tasks) + +If the worker process dies unexpectedly: + +- Task remains in PROGRESS state with stale `worker_pid` +- Other workers periodically check for orphaned tasks +- Dead tasks are detected via `os.kill(pid, 0)` +- Status set to `FAILED` with "worker process no longer running" error + +### 4. Subprocess Exit + +If subprocess exits with non-zero code (without exception): + +- Status set to `FAILED` +- Exit code recorded in `error` field + +## Checking Task Status + +### In Code + +```python +from django_simple_queue.models import Task + +task = Task.objects.get(id=task_id) + +if task.status == Task.QUEUED: + print("Still waiting...") +elif task.status == Task.PROGRESS: + print(f"Running on PID {task.worker_pid}") +elif task.status == Task.COMPLETED: + print(f"Done! Output: {task.output}") +elif task.status == Task.FAILED: + print(f"Failed: {task.error}") +``` + +### Via HTTP + +```bash +# JSON response +curl "http://localhost:8000/django_simple_queue/task?task_id=UUID&type=json" +``` + +```json +{ + "id": "abc123...", + "status": "In progress", + "output": null, + "error": null, + "worker_pid": 12345, + "log": null +} +``` + +## Re-queuing Failed Tasks + +Failed tasks can be re-queued through the admin or code: + +```python +# Re-queue a single task +task = Task.objects.get(id=task_id) +task.status = Task.QUEUED +task.error = None +task.worker_pid = None +task.save() + +# Re-queue all failed tasks +Task.objects.filter(status=Task.FAILED).update( + status=Task.QUEUED, + error=None, + worker_pid=None +) +``` + +Or use the "Enqueue" action in Django Admin. + +## Task Output Fields + +After a task completes, several fields contain useful information: + +```python +task = Task.objects.get(id=task_id) + +# Return value from the task function +print(task.output) + +# Exception traceback (if failed) +print(task.error) + +# Captured stdout/stderr/logging output +print(task.log) +``` + +## Next Steps + +- Use [signals](signals.md) to react to lifecycle events +- Handle [errors](errors.md) gracefully +- Learn about [generator functions](generators.md) for streaming output diff --git a/docs/guides/signals.md b/docs/guides/signals.md new file mode 100644 index 0000000..d00f8c5 --- /dev/null +++ b/docs/guides/signals.md @@ -0,0 +1,246 @@ +# Using Signals + +Django Simple Queue emits signals at various points during task execution, allowing you to hook into the task lifecycle. + +## Available Signals + +| Signal | Fired When | Arguments | +|--------|------------|-----------| +| `before_job` | Before task execution starts | `task` | +| `on_success` | Task completes successfully | `task` | +| `on_failure` | Task fails (exception, timeout, crash) | `task`, `error` | +| `before_loop` | Before each generator iteration | `task`, `iteration` | +| `after_loop` | After each generator iteration | `task`, `output`, `iteration` | + +## Connecting to Signals + +### Using the Decorator + +```python +# myapp/signals.py +from django.dispatch import receiver +from django_simple_queue.signals import before_job, on_success, on_failure + +@receiver(before_job) +def log_task_start(sender, task, **kwargs): + """Log when a task starts executing.""" + print(f"Starting task {task.id}: {task.task}") + + +@receiver(on_success) +def handle_task_success(sender, task, **kwargs): + """Handle successful task completion.""" + print(f"Task {task.id} completed: {task.output}") + + # Example: Send notification + if task.task == "myapp.tasks.generate_report": + notify_user_report_ready(task) + + +@receiver(on_failure) +def handle_task_failure(sender, task, error, **kwargs): + """Handle task failure.""" + print(f"Task {task.id} failed: {error}") + + # Example: Alert on critical failures + if "payment" in task.task: + alert_ops_team(task, error) +``` + +### Manual Connection + +```python +from django_simple_queue.signals import on_success + +def my_handler(sender, task, **kwargs): + print(f"Task {task.id} done") + +on_success.connect(my_handler) +``` + +## Signal Details + +### before_job + +Fired in the subprocess just before the task function is called. + +```python +@receiver(before_job) +def on_before_job(sender, task, **kwargs): + """ + Args: + sender: Task class + task: The Task instance about to execute + """ + # Good for: logging, setting up context + logger.info(f"Starting {task.task} with args: {task.args}") +``` + +### on_success + +Fired after the task function returns successfully (no exception). + +```python +@receiver(on_success) +def on_task_success(sender, task, **kwargs): + """ + Args: + sender: Task class + task: The completed Task instance (status=COMPLETED) + """ + # Good for: notifications, triggering follow-up tasks + if task.output: + process_result(task.output) +``` + +### on_failure + +Fired when a task fails for any reason. + +```python +@receiver(on_failure) +def on_task_failure(sender, task, error, **kwargs): + """ + Args: + sender: Task class + task: The failed Task instance (status=FAILED) + error: The exception that caused failure, or None for: + - Orphaned tasks (worker died) + - Subprocess non-zero exit + """ + if error: + logger.exception(f"Task {task.id} raised: {error}") + else: + logger.error(f"Task {task.id} failed without exception: {task.error}") +``` + +### before_loop / after_loop + +For generator functions, these fire on each iteration: + +```python +@receiver(before_loop) +def on_before_iteration(sender, task, iteration, **kwargs): + """ + Args: + sender: Task class + task: The Task instance + iteration: 0-based iteration index + """ + logger.debug(f"Task {task.id} starting iteration {iteration}") + + +@receiver(after_loop) +def on_after_iteration(sender, task, output, iteration, **kwargs): + """ + Args: + sender: Task class + task: The Task instance + output: The value yielded by the generator + iteration: 0-based iteration index + """ + logger.debug(f"Task {task.id} iteration {iteration} yielded: {output}") +``` + +## Loading Signal Handlers + +Ensure your signal handlers are loaded when Django starts. Add to your app's `apps.py`: + +```python +# myapp/apps.py +from django.apps import AppConfig + +class MyAppConfig(AppConfig): + name = 'myapp' + + def ready(self): + import myapp.signals # noqa: F401 +``` + +## Common Use Cases + +### Retry Failed Tasks + +```python +from django_simple_queue.signals import on_failure +from django_simple_queue.utils import create_task +from django_simple_queue.models import Task + +@receiver(on_failure) +def auto_retry(sender, task, error, **kwargs): + """Automatically retry failed tasks up to 3 times.""" + import json + + args = json.loads(task.args) if task.args else {} + retry_count = args.get('_retry_count', 0) + + if retry_count < 3: + create_task( + task=task.task, + args={**args, '_retry_count': retry_count + 1} + ) +``` + +### Metrics/Monitoring + +```python +from django_simple_queue.signals import before_job, on_success, on_failure +import time + +task_start_times = {} + +@receiver(before_job) +def record_start(sender, task, **kwargs): + task_start_times[str(task.id)] = time.time() + +@receiver(on_success) +def record_success_metrics(sender, task, **kwargs): + duration = time.time() - task_start_times.pop(str(task.id), time.time()) + metrics.histogram('task.duration', duration, tags=[f'task:{task.task}']) + metrics.increment('task.success', tags=[f'task:{task.task}']) + +@receiver(on_failure) +def record_failure_metrics(sender, task, error, **kwargs): + task_start_times.pop(str(task.id), None) + metrics.increment('task.failure', tags=[f'task:{task.task}']) +``` + +### Chain Tasks + +```python +from django_simple_queue.signals import on_success +from django_simple_queue.utils import create_task +import json + +@receiver(on_success) +def chain_tasks(sender, task, **kwargs): + """Run follow-up tasks based on completed task.""" + if task.task == "myapp.tasks.process_order": + args = json.loads(task.args) + order_id = args['order_id'] + + # Queue follow-up tasks + create_task( + task="myapp.tasks.send_confirmation_email", + args={"order_id": order_id} + ) + create_task( + task="myapp.tasks.update_inventory", + args={"order_id": order_id} + ) +``` + +## Important Notes + +1. **Signals run in subprocess**: `before_job`, `on_success`, `on_failure`, `before_loop`, and `after_loop` run in the task subprocess, not the main worker process. + +2. **Database transactions**: Signal handlers run in the same transaction as the task status update. + +3. **Exceptions in handlers**: Exceptions in signal handlers are logged but don't affect the task status. + +4. **Order not guaranteed**: If multiple handlers are connected, execution order is not guaranteed. + +## Next Steps + +- Learn about [generator functions](generators.md) and loop signals +- Handle [errors](errors.md) in your tasks diff --git a/docs/guides/worker-optimization.md b/docs/guides/worker-optimization.md new file mode 100644 index 0000000..ff4d7b5 --- /dev/null +++ b/docs/guides/worker-optimization.md @@ -0,0 +1,295 @@ +# Worker Optimization + +This guide covers how to optimize worker memory usage and performance for production deployments. + +## The Problem + +By default, when you run `python manage.py task_worker`, Django loads your entire application including: + +- All installed apps +- All middleware +- Template engines +- Static file handlers +- Authentication backends +- etc. + +This can result in workers using 400-500 MB of RAM each, with 10-20 subprocesses. + +## The Solution: Lean Settings + +Create a dedicated settings file that only loads what the worker needs: + +```python +# myproject/task_worker_settings.py +from .settings import * + +# Only essential apps +INSTALLED_APPS = [ + "django.contrib.contenttypes", # Required for models + "django_simple_queue", + "myapp", # Your app with task functions +] + +# No middleware needed +MIDDLEWARE = [] + +# No templates needed +TEMPLATES = [] + +# Disable static/media files +STATICFILES_DIRS = () +STATIC_URL = None +STATIC_ROOT = None +MEDIA_ROOT = None +MEDIA_URL = None + +# Disable i18n if not needed +USE_I18N = False +USE_TZ = True # Keep if tasks rely on timezones + +# Optimize database connections +DATABASES["default"]["CONN_MAX_AGE"] = None # Persistent connections +DATABASES["default"]["OPTIONS"] = { + "connect_timeout": 10, +} + +# Disable auth validators +AUTH_PASSWORD_VALIDATORS = [] + +# Disable admin +ADMIN_ENABLED = False + +# No URL routing needed +ROOT_URLCONF = None +``` + +## Running with Lean Settings + +Use the `DJANGO_SETTINGS_MODULE` environment variable: + +```bash +DJANGO_SETTINGS_MODULE=myproject.task_worker_settings python manage.py task_worker +``` + +Or set it in your process manager configuration. + +## Results + +With lean settings, you can expect: + +| Metric | Default Settings | Lean Settings | +|--------|------------------|---------------| +| RAM per worker | 400-500 MB | 30-50 MB | +| Subprocess count | 10-20 | 1 | +| Startup time | Slower | Faster | + +## Which Apps to Include + +Include only apps that: + +1. Define models used by your tasks +2. Contain your task functions +3. Are dependencies of the above + +```python +INSTALLED_APPS = [ + "django.contrib.contenttypes", # Always required + "django_simple_queue", # The queue itself + + # Only your apps that tasks actually use + "myapp.orders", # If tasks access Order model + "myapp.emails", # If tasks send emails + # "myapp.frontend", # Skip - not used by tasks + # "myapp.admin", # Skip - not used by tasks +] +``` + +## Process Manager Configuration + +### systemd + +```ini +# /etc/systemd/system/task_worker.service +[Unit] +Description=Django Simple Queue Worker +After=network.target + +[Service] +Type=simple +User=www-data +WorkingDirectory=/var/www/myproject +Environment=DJANGO_SETTINGS_MODULE=myproject.task_worker_settings +ExecStart=/var/www/myproject/venv/bin/python manage.py task_worker +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +``` + +### Supervisor + +```ini +# /etc/supervisor/conf.d/task_worker.conf +[program:task_worker] +command=/var/www/myproject/venv/bin/python manage.py task_worker +directory=/var/www/myproject +user=www-data +environment=DJANGO_SETTINGS_MODULE="myproject.task_worker_settings" +autostart=true +autorestart=true +redirect_stderr=true +stdout_logfile=/var/log/task_worker.log +``` + +### Docker + +```dockerfile +# Dockerfile.worker +FROM python:3.11 + +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . + +ENV DJANGO_SETTINGS_MODULE=myproject.task_worker_settings + +CMD ["python", "manage.py", "task_worker"] +``` + +```yaml +# docker-compose.yml +services: + worker: + build: + context: . + dockerfile: Dockerfile.worker + environment: + - DJANGO_SETTINGS_MODULE=myproject.task_worker_settings + depends_on: + - db + restart: always +``` + +## Multiple Workers + +For parallel task processing, run multiple worker instances: + +```bash +# Run 4 workers +for i in {1..4}; do + DJANGO_SETTINGS_MODULE=myproject.task_worker_settings \ + python manage.py task_worker & +done +``` + +Each worker polls independently and uses `SELECT FOR UPDATE SKIP LOCKED` to avoid processing the same task. + +### With systemd (multiple instances) + +```ini +# /etc/systemd/system/task_worker@.service +[Unit] +Description=Django Task Worker %i +After=network.target + +[Service] +Type=simple +User=www-data +WorkingDirectory=/var/www/myproject +Environment=DJANGO_SETTINGS_MODULE=myproject.task_worker_settings +ExecStart=/var/www/myproject/venv/bin/python manage.py task_worker +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +```bash +# Enable 4 worker instances +sudo systemctl enable task_worker@{1..4} +sudo systemctl start task_worker@{1..4} +``` + +## Monitoring Workers + +### Check Memory Usage + +The worker logs memory usage on each heartbeat: + +``` +2024-01-15 10:30:00: [RAM Usage: 45.2 MB] Heartbeat.. +``` + +### Check Worker Status + +```bash +# systemd +sudo systemctl status task_worker + +# supervisor +sudo supervisorctl status task_worker + +# docker +docker-compose logs -f worker +``` + +### Monitor Task Queue + +```python +from django_simple_queue.models import Task + +# Queue depth +queued = Task.objects.filter(status=Task.QUEUED).count() + +# In-progress tasks +in_progress = Task.objects.filter(status=Task.PROGRESS).count() + +# Failed in last hour +from django.utils import timezone +from datetime import timedelta + +recent_failures = Task.objects.filter( + status=Task.FAILED, + modified__gte=timezone.now() - timedelta(hours=1) +).count() +``` + +## Troubleshooting + +### ImportError in Worker + +If tasks fail with `ImportError`, ensure the required app is in `INSTALLED_APPS`: + +```python +# task_worker_settings.py +INSTALLED_APPS = [ + ... + "myapp.payments", # Add the app your task needs +] +``` + +### Database Connection Issues + +With persistent connections, stale connections can cause issues: + +```python +# task_worker_settings.py +DATABASES["default"]["CONN_MAX_AGE"] = 600 # 10 minutes instead of unlimited +DATABASES["default"]["CONN_HEALTH_CHECKS"] = True # Django 4.1+ +``` + +### High Memory with Lean Settings + +If memory is still high, check: + +1. Your task functions aren't loading heavy modules at import time +2. You're not importing from apps not in `INSTALLED_APPS` +3. Consider using `gc.collect()` after large tasks + +## Next Steps + +- Configure [task timeouts](../getting-started/configuration.md) +- Use [PostgreSQL](../advanced/databases.md) for production +- Monitor tasks via the [Admin interface](../advanced/admin.md) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..58ae869 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,65 @@ +# Django Simple Queue + +A lightweight, database-backed task queue for Django applications. + +## Overview + +Django Simple Queue provides a simple way to run background tasks in Django using your existing database as the message broker. No additional infrastructure like Redis or RabbitMQ is required. + +## Features + +- **Database-backed**: Uses your existing Django database as the queue broker +- **Subprocess isolation**: Each task runs in its own subprocess for memory safety +- **Generator support**: Stream output from long-running tasks with generator functions +- **Lifecycle signals**: Hook into task execution with Django signals +- **Orphan detection**: Automatically detects and handles crashed worker processes +- **Task timeout**: Configurable timeout to prevent runaway tasks +- **Admin interface**: View and manage tasks through Django admin +- **Allowlist security**: Restrict which callables can be executed + +## Quick Example + +```python +# myapp/tasks.py +def send_welcome_email(user_id, template="default"): + from myapp.models import User + user = User.objects.get(id=user_id) + # Send email... + return f"Email sent to {user.email}" + +# Enqueue a task +from django_simple_queue.utils import create_task + +task_id = create_task( + task="myapp.tasks.send_welcome_email", + args={"user_id": 42, "template": "welcome"} +) +``` + +```bash +# Start the worker +python manage.py task_worker +``` + +## When to Use + +Django Simple Queue is ideal for: + +- **Small to medium applications** that don't need the complexity of Celery +- **Development environments** where you want a simple queue without extra services +- **Applications already using PostgreSQL** that can leverage its locking features +- **Teams that want minimal operational overhead** + +For high-throughput applications or those requiring advanced features like task chaining, routing, or rate limiting, consider [Celery](https://docs.celeryq.dev/) or [Django-Q2](https://django-q2.readthedocs.io/). + +## Installation + +```bash +pip install django-simple-queue +``` + +See the [Installation Guide](getting-started/installation.md) for detailed setup instructions. + +## License + +MIT License diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 0000000..929380b --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,109 @@ +# Configuration + +Configuration functions for reading Django Simple Queue settings. + +## API Reference + +::: django_simple_queue.conf + options: + show_source: true + +## Settings Overview + +All settings are read from Django's `settings.py` with the `DJANGO_SIMPLE_QUEUE_` prefix. + +| Setting | Default | Description | +|---------|---------|-------------| +| `DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS` | `None` | Set of allowed task paths | +| `DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT` | `3600` | Task timeout in seconds | +| `DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE` | `10MB` | Max output size in bytes | +| `DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE` | `1MB` | Max args JSON size in bytes | + +## Functions + +### get_allowed_tasks() + +Returns the set of allowed task callables, or `None` if no restriction. + +```python +from django_simple_queue.conf import get_allowed_tasks + +allowed = get_allowed_tasks() +if allowed is None: + print("All tasks allowed (not recommended)") +else: + print(f"Allowed tasks: {allowed}") +``` + +### is_task_allowed(task_path) + +Check if a specific task path is allowed. + +```python +from django_simple_queue.conf import is_task_allowed + +if is_task_allowed("myapp.tasks.send_email"): + print("Task is allowed") +else: + print("Task is blocked") +``` + +### get_task_timeout() + +Returns the task timeout in seconds, or `None` if disabled. + +```python +from django_simple_queue.conf import get_task_timeout + +timeout = get_task_timeout() +if timeout: + print(f"Tasks timeout after {timeout} seconds") +else: + print("No timeout configured") +``` + +### get_max_output_size() + +Returns the maximum output size in bytes. + +```python +from django_simple_queue.conf import get_max_output_size + +max_size = get_max_output_size() +print(f"Max output: {max_size / 1024 / 1024:.1f} MB") +``` + +### get_max_args_size() + +Returns the maximum args JSON size in bytes. + +```python +from django_simple_queue.conf import get_max_args_size + +max_size = get_max_args_size() +print(f"Max args: {max_size / 1024:.1f} KB") +``` + +## Example Configuration + +```python +# settings.py + +# Only allow specific tasks +DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS = { + "orders.tasks.process_order", + "emails.tasks.send_notification", + "reports.tasks.generate_daily", +} + +# 5 minute timeout +DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT = 300 + +# Limit output to 5 MB +DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE = 5 * 1024 * 1024 + +# Limit args to 100 KB +DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE = 100 * 1024 +``` + +See the [Configuration Guide](../getting-started/configuration.md) for detailed explanations of each setting. diff --git a/docs/reference/models.md b/docs/reference/models.md new file mode 100644 index 0000000..f7912c3 --- /dev/null +++ b/docs/reference/models.md @@ -0,0 +1,101 @@ +# Task Model + +The `Task` model represents a unit of work to be executed asynchronously by a worker process. + +## API Reference + +::: django_simple_queue.models.Task + options: + show_source: true + members: + - QUEUED + - PROGRESS + - COMPLETED + - FAILED + - CANCELLED + - STATUS_CHOICES + - as_dict + - clean_task + - clean_args + +## Status Constants + +| Constant | Value | Description | +|----------|-------|-------------| +| `Task.QUEUED` | 0 | Task is waiting to be picked up | +| `Task.PROGRESS` | 1 | Task is currently executing | +| `Task.COMPLETED` | 2 | Task finished successfully | +| `Task.FAILED` | 3 | Task encountered an error | +| `Task.CANCELLED` | 4 | Task was manually cancelled | + +## Fields + +| Field | Type | Description | +|-------|------|-------------| +| `id` | UUID | Primary key (auto-generated) | +| `created` | DateTime | When the task was created | +| `modified` | DateTime | When the task was last updated | +| `task` | CharField | Dotted path to the callable | +| `args` | TextField | JSON-encoded keyword arguments | +| `status` | IntegerField | Current status (see constants above) | +| `output` | TextField | Return value from the task function | +| `worker_pid` | IntegerField | PID of the worker process | +| `error` | TextField | Error message and traceback | +| `log` | TextField | Captured stdout/stderr/logging | + +## Usage Examples + +### Query Tasks by Status + +```python +from django_simple_queue.models import Task + +# Get all pending tasks +pending = Task.objects.filter(status=Task.QUEUED) + +# Get failed tasks from today +from django.utils import timezone +from datetime import timedelta + +today = timezone.now().date() +failed_today = Task.objects.filter( + status=Task.FAILED, + created__date=today +) +``` + +### Check Task Result + +```python +task = Task.objects.get(id=task_id) + +if task.status == Task.COMPLETED: + print(f"Result: {task.output}") +elif task.status == Task.FAILED: + print(f"Error: {task.error}") + print(f"Logs: {task.log}") +``` + +### Re-queue a Failed Task + +```python +task = Task.objects.get(id=task_id) +task.status = Task.QUEUED +task.error = None +task.worker_pid = None +task.save() +``` + +### Get JSON Representation + +```python +task = Task.objects.get(id=task_id) +data = task.as_dict +# { +# "id": "...", +# "created": "...", +# "status": "Completed", +# "output": "...", +# ... +# } +``` diff --git a/docs/reference/monitor.md b/docs/reference/monitor.md new file mode 100644 index 0000000..d6220aa --- /dev/null +++ b/docs/reference/monitor.md @@ -0,0 +1,116 @@ +# Monitoring + +Functions for monitoring task health and handling failures. + +## API Reference + +::: django_simple_queue.monitor + options: + show_source: true + +## Functions + +### detect_orphaned_tasks() + +Scans for tasks whose worker processes have died and marks them as failed. + +This function is called automatically by the task worker on each polling cycle. It: + +1. Finds all tasks with status `PROGRESS` +2. Checks if each task's `worker_pid` is still alive +3. If the process is dead, marks the task as `FAILED` +4. Fires the `on_failure` signal + +```python +from django_simple_queue.monitor import detect_orphaned_tasks + +# Usually called by the worker, but can be called manually +detect_orphaned_tasks() +``` + +### handle_subprocess_exit(task_id, exit_code) + +Handles non-zero subprocess exit codes. + +Called by the worker when a task subprocess exits with a non-zero code but didn't raise an exception (e.g., killed by signal). + +```python +from django_simple_queue.monitor import handle_subprocess_exit + +# Called internally by the worker +handle_subprocess_exit(task_id, exit_code=1) +``` + +### handle_task_timeout(task_id, timeout_seconds) + +Marks a task as failed due to timeout. + +Called by the worker when a task exceeds `DJANGO_SIMPLE_QUEUE_TASK_TIMEOUT`. + +```python +from django_simple_queue.monitor import handle_task_timeout + +# Called internally by the worker +handle_task_timeout(task_id, timeout_seconds=300) +``` + +## How Orphan Detection Works + +``` +Worker Process A Worker Process B + │ │ + │ Claims Task T1 │ + │ Sets status=PROGRESS │ + │ Sets worker_pid=A │ + │ │ + │ Starts executing... │ + │ │ + X Worker A crashes! │ + │ + │ Polls for tasks... + │ Calls detect_orphaned_tasks() + │ Finds T1 with status=PROGRESS + │ Checks: is PID A alive? + │ os.kill(A, 0) → ProcessLookupError + │ Marks T1 as FAILED + │ Fires on_failure signal +``` + +## Failure Messages + +| Scenario | Error Message | +|----------|---------------| +| Worker crash | "Task failed: worker process (PID X) no longer running" | +| Timeout | "Task timed out after X seconds" | +| Non-zero exit | "Worker subprocess exited with code X" | + +## Monitoring in Production + +### Check for Orphaned Tasks + +```python +from django_simple_queue.models import Task + +# Tasks that might be orphaned (in progress for too long) +from django.utils import timezone +from datetime import timedelta + +stale = Task.objects.filter( + status=Task.PROGRESS, + modified__lt=timezone.now() - timedelta(hours=1) +) +for task in stale: + print(f"Possibly orphaned: {task.id} (PID: {task.worker_pid})") +``` + +### Cleanup Script + +```python +from django_simple_queue.monitor import detect_orphaned_tasks + +# Run periodically as a cron job or management command +detect_orphaned_tasks() +print("Orphan detection complete") +``` + +See the [Task Lifecycle](../guides/lifecycle.md) guide for more on failure handling. diff --git a/docs/reference/signals.md b/docs/reference/signals.md new file mode 100644 index 0000000..55f04ae --- /dev/null +++ b/docs/reference/signals.md @@ -0,0 +1,112 @@ +# Signals + +Signals emitted during task lifecycle. Connect to these signals to hook into task execution. + +## API Reference + +::: django_simple_queue.signals + options: + show_source: true + +## Signal Summary + +| Signal | When Fired | Arguments | +|--------|------------|-----------| +| `before_job` | Before task execution | `task` | +| `on_success` | Task completed successfully | `task` | +| `on_failure` | Task failed | `task`, `error` | +| `before_loop` | Before generator iteration | `task`, `iteration` | +| `after_loop` | After generator iteration | `task`, `output`, `iteration` | + +## Connecting to Signals + +### Using Decorator + +```python +from django.dispatch import receiver +from django_simple_queue.signals import on_success, on_failure + +@receiver(on_success) +def handle_success(sender, task, **kwargs): + print(f"Task {task.id} completed") + +@receiver(on_failure) +def handle_failure(sender, task, error, **kwargs): + print(f"Task {task.id} failed: {error}") +``` + +### Manual Connection + +```python +from django_simple_queue.signals import before_job + +def my_handler(sender, task, **kwargs): + print(f"Starting {task.task}") + +before_job.connect(my_handler) +``` + +## Signal Arguments + +### before_job + +```python +@receiver(before_job) +def handler(sender, task, **kwargs): + # sender: Task class + # task: Task instance about to execute + pass +``` + +### on_success + +```python +@receiver(on_success) +def handler(sender, task, **kwargs): + # sender: Task class + # task: Completed Task instance + pass +``` + +### on_failure + +```python +@receiver(on_failure) +def handler(sender, task, error, **kwargs): + # sender: Task class + # task: Failed Task instance + # error: Exception or None (for orphaned/timeout) + pass +``` + +### before_loop / after_loop + +```python +@receiver(before_loop) +def handler(sender, task, iteration, **kwargs): + # iteration: 0-based index + pass + +@receiver(after_loop) +def handler(sender, task, output, iteration, **kwargs): + # output: Value yielded by generator + # iteration: 0-based index + pass +``` + +## Loading Signal Handlers + +Ensure handlers load on Django startup: + +```python +# myapp/apps.py +from django.apps import AppConfig + +class MyAppConfig(AppConfig): + name = 'myapp' + + def ready(self): + import myapp.signals # noqa +``` + +See the [Using Signals](../guides/signals.md) guide for more examples. diff --git a/docs/reference/utils.md b/docs/reference/utils.md new file mode 100644 index 0000000..0abaf9e --- /dev/null +++ b/docs/reference/utils.md @@ -0,0 +1,120 @@ +# Utilities + +Utility functions for creating and managing tasks. + +## API Reference + +::: django_simple_queue.utils + options: + show_source: true + members: + - create_task + - TaskNotAllowedError + +## create_task + +The primary way to enqueue tasks for background execution. + +### Signature + +```python +def create_task(task: str, args: dict) -> UUID: + ... +``` + +### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `task` | str | Dotted path to the callable (e.g., "myapp.tasks.send_email") | +| `args` | dict | Keyword arguments to pass to the callable | + +### Returns + +- `UUID`: The unique identifier of the created task + +### Raises + +- `TypeError`: If `args` is not a dictionary +- `TaskNotAllowedError`: If task is not in `DJANGO_SIMPLE_QUEUE_ALLOWED_TASKS` + +### Examples + +```python +from django_simple_queue.utils import create_task + +# Basic usage +task_id = create_task( + task="myapp.tasks.send_email", + args={"to": "user@example.com", "subject": "Hello"} +) + +# With complex arguments +task_id = create_task( + task="myapp.tasks.process_order", + args={ + "order_id": 12345, + "options": { + "notify": True, + "priority": "high" + } + } +) + +# Check the created task +from django_simple_queue.models import Task +task = Task.objects.get(id=task_id) +print(task.status) # 0 (QUEUED) +``` + +## TaskNotAllowedError + +Exception raised when attempting to create a task that is not in the allowlist. + +### Example + +```python +from django_simple_queue.utils import create_task, TaskNotAllowedError + +try: + task_id = create_task( + task="some.unknown.function", + args={} + ) +except TaskNotAllowedError as e: + print(f"Task not allowed: {e}") +``` + +## Best Practices + +### Pass IDs, Not Objects + +```python +# Good +create_task( + task="myapp.tasks.process_user", + args={"user_id": user.id} +) + +# Bad - can't serialize model instances +create_task( + task="myapp.tasks.process_user", + args={"user": user} # TypeError! +) +``` + +### Use JSON-Serializable Arguments + +```python +# Good +create_task( + task="myapp.tasks.schedule", + args={"date": "2024-01-15T10:00:00"} +) + +# Bad - datetime not JSON-serializable by default +create_task( + task="myapp.tasks.schedule", + args={"date": datetime.now()} # Error! +) +``` diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..e3eb84c --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,87 @@ +site_name: Django Simple Queue +site_url: https://d3banjan.github.io/django-simple-queue/ +repo_url: https://github.com/d3banjan/django-simple-queue +repo_name: d3banjan/django-simple-queue +site_description: A Django-based task queue using the database as the broker + +theme: + name: material + palette: + - scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - content.code.copy + - content.code.annotate + - navigation.sections + - navigation.expand + - navigation.top + - search.highlight + - search.suggest + - toc.follow + +plugins: + - search + - mkdocstrings: + handlers: + python: + options: + docstring_style: google + show_source: true + show_root_heading: true + show_root_full_path: false + show_symbol_type_heading: true + show_symbol_type_toc: true + members_order: source + merge_init_into_class: true + show_signature_annotations: true + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - admonition + - pymdownx.details + - pymdownx.tabbed: + alternate_style: true + - attr_list + - md_in_html + - tables + - toc: + permalink: true + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Configuration: getting-started/configuration.md + - Quick Start: getting-started/quickstart.md + - Guides: + - Creating Tasks: guides/creating-tasks.md + - Task Lifecycle: guides/lifecycle.md + - Using Signals: guides/signals.md + - Generator Functions: guides/generators.md + - Error Handling: guides/errors.md + - Worker Optimization: guides/worker-optimization.md + - Reference: + - Task Model: reference/models.md + - Signals: reference/signals.md + - Utilities: reference/utils.md + - Configuration: reference/config.md + - Monitoring: reference/monitor.md + - Advanced: + - Database Backends: advanced/databases.md + - Admin Interface: advanced/admin.md From 1350397cf062140c7efe0f96a11cc689419636a5 Mon Sep 17 00:00:00 2001 From: Debanjan Basu Date: Wed, 28 Jan 2026 23:43:23 +0530 Subject: [PATCH 14/14] fix: add type annotations to silence griffe warnings in strict mode Co-Authored-By: Claude Opus 4.5 --- django_simple_queue/admin.py | 9 +++++++-- django_simple_queue/conf.py | 12 +++++++----- django_simple_queue/models.py | 2 +- django_simple_queue/monitor.py | 9 ++++++--- django_simple_queue/utils.py | 5 ++++- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/django_simple_queue/admin.py b/django_simple_queue/admin.py index c63ebf4..7f9ea09 100644 --- a/django_simple_queue/admin.py +++ b/django_simple_queue/admin.py @@ -3,9 +3,14 @@ Provides a customized admin interface for viewing and managing tasks. """ +from __future__ import annotations + from django.contrib import admin, messages +from django.db.models import QuerySet +from django.http import HttpRequest from django.urls import reverse from django.utils.html import format_html +from django.utils.safestring import SafeString from django.utils.translation import ngettext from django_simple_queue.models import Task @@ -33,7 +38,7 @@ def get_readonly_fields(self, request, obj=None): self.readonly_fields = [field.name for field in obj.__class__._meta.fields] return self.readonly_fields - def status_page_link(self, obj): + def status_page_link(self, obj: Task) -> SafeString: """ Generate a clickable link to the task status page. @@ -54,7 +59,7 @@ def status_page_link(self, obj): status_page_link.short_description = "Status" @admin.action(description='Enqueue') - def enqueue_tasks(self, request, queryset): + def enqueue_tasks(self, request: HttpRequest, queryset: QuerySet[Task]) -> None: """ Admin action to re-queue selected tasks. diff --git a/django_simple_queue/conf.py b/django_simple_queue/conf.py index ac0437d..6a771bc 100644 --- a/django_simple_queue/conf.py +++ b/django_simple_queue/conf.py @@ -3,10 +3,12 @@ Settings are read from Django's settings.py with the DJANGO_SIMPLE_QUEUE_ prefix. """ +from __future__ import annotations + from django.conf import settings -def get_allowed_tasks(): +def get_allowed_tasks() -> set[str] | None: """ Returns the set of allowed task callables. @@ -25,7 +27,7 @@ def get_allowed_tasks(): return set(allowed) -def is_task_allowed(task_path): +def is_task_allowed(task_path: str) -> bool: """ Check if a task path is in the allowed list. @@ -43,7 +45,7 @@ def is_task_allowed(task_path): return task_path in allowed -def get_max_output_size(): +def get_max_output_size() -> int: """ Returns the maximum allowed output size in bytes. @@ -55,7 +57,7 @@ def get_max_output_size(): return getattr(settings, "DJANGO_SIMPLE_QUEUE_MAX_OUTPUT_SIZE", 10 * 1024 * 1024) -def get_max_args_size(): +def get_max_args_size() -> int: """ Returns the maximum allowed args JSON size in bytes. @@ -67,7 +69,7 @@ def get_max_args_size(): return getattr(settings, "DJANGO_SIMPLE_QUEUE_MAX_ARGS_SIZE", 1024 * 1024) -def get_task_timeout(): +def get_task_timeout() -> int | None: """ Returns the maximum execution time for a task in seconds. diff --git a/django_simple_queue/models.py b/django_simple_queue/models.py index 42973a5..62b2089 100644 --- a/django_simple_queue/models.py +++ b/django_simple_queue/models.py @@ -76,7 +76,7 @@ class Meta: verbose_name_plural = _("Tasks") @property - def as_dict(self): + def as_dict(self) -> dict: """ Returns a dictionary representation of the task. diff --git a/django_simple_queue/monitor.py b/django_simple_queue/monitor.py index 6aba0fa..589eaa5 100644 --- a/django_simple_queue/monitor.py +++ b/django_simple_queue/monitor.py @@ -5,7 +5,10 @@ processes have died unexpectedly, as well as handling task timeouts and subprocess failures. """ +from __future__ import annotations + import os +import uuid from django.db import transaction @@ -13,7 +16,7 @@ from django_simple_queue.models import Task -def detect_orphaned_tasks(): +def detect_orphaned_tasks() -> None: """ Detect and mark orphaned tasks as failed. @@ -50,7 +53,7 @@ def detect_orphaned_tasks(): pass # PID exists, different user — worker is alive -def handle_subprocess_exit(task_id, exit_code): +def handle_subprocess_exit(task_id: uuid.UUID, exit_code: int | None) -> None: """ Handle a task subprocess that exited with a non-zero code. @@ -72,7 +75,7 @@ def handle_subprocess_exit(task_id, exit_code): signals.on_failure.send(sender=Task, task=task, error=None) -def handle_task_timeout(task_id, timeout_seconds): +def handle_task_timeout(task_id: uuid.UUID, timeout_seconds: int) -> None: """ Mark a task as failed due to exceeding the timeout. diff --git a/django_simple_queue/utils.py b/django_simple_queue/utils.py index eaec65d..5b83e5a 100644 --- a/django_simple_queue/utils.py +++ b/django_simple_queue/utils.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import json +import uuid from django_simple_queue.conf import is_task_allowed, get_allowed_tasks from django_simple_queue.models import Task @@ -9,7 +12,7 @@ class TaskNotAllowedError(Exception): pass -def create_task(task, args): +def create_task(task: str, args: dict) -> uuid.UUID: """ Create a new task to be executed by the worker.