From ddb7236b0ae9be3163c90f799fb79396e9f61cc8 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Wed, 3 Dec 2025 16:57:47 +0000 Subject: [PATCH 1/3] Refs #36520 -- Removed release note for refactored `parse_header_parameters`. --- docs/releases/6.0.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/releases/6.0.txt b/docs/releases/6.0.txt index 89786c7ccb62..de874ad85586 100644 --- a/docs/releases/6.0.txt +++ b/docs/releases/6.0.txt @@ -478,10 +478,6 @@ Miscellaneous * The :ref:`JSON ` serializer now writes a newline at the end of the output, even without the ``indent`` option set. -* The undocumented ``django.utils.http.parse_header_parameters()`` function is - refactored to use Python's :class:`email.message.Message` for parsing. - Input headers exceeding 10000 characters will now raise :exc:`ValueError`. - * The minimum supported version of ``asgiref`` is increased from 3.8.1 to 3.9.1. From 93540b34d4ef46f68df2c8bfe90447d0f649a852 Mon Sep 17 00:00:00 2001 From: rimi0108 Date: Sat, 4 Oct 2025 13:52:59 +0900 Subject: [PATCH 2/3] Fixed #35729 -- Enabled natural key serialization opt-out for subclasses. Refactored serialization logic to allow models inheriting a natural_key() method (e.g. AbstractBaseUser) to explicitly opt out of natural key serialization by returning an empty tuple from the method. Thanks Jonas Dittrich for the report. Co-authored-by: Jacob Walls --- django/core/serializers/base.py | 28 ++++++++++++++++++ django/core/serializers/python.py | 21 +++++++------- django/core/serializers/xml_serializer.py | 33 +++++++++++---------- docs/releases/6.1.txt | 5 +++- docs/topics/serialization.txt | 9 ++++++ tests/serializers/models/natural.py | 17 +++++++++++ tests/serializers/test_natural.py | 35 +++++++++++++++++++++++ 7 files changed, 119 insertions(+), 29 deletions(-) diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py index efc55981ebbc..cc9caf9a82b9 100644 --- a/django/core/serializers/base.py +++ b/django/core/serializers/base.py @@ -209,6 +209,34 @@ def getvalue(self): if callable(getattr(self.stream, "getvalue", None)): return self.stream.getvalue() + def _resolve_natural_key(self, obj): + """Return a natural key tuple for the given object when available.""" + try: + return obj.natural_key() + except AttributeError: + return None + + def _resolve_fk_natural_key(self, obj, field): + """ + Return the natural key for a ForeignKey's related object, or None if + not supported. + """ + if not self._model_supports_natural_key(field.remote_field.model): + return None + + related = getattr(obj, field.name, None) + try: + return related.natural_key() + except AttributeError: + return None + + def _model_supports_natural_key(self, model): + """Return True if the model defines a natural_key() method.""" + try: + return callable(model.natural_key) + except AttributeError: + return False + class Deserializer: """ diff --git a/django/core/serializers/python.py b/django/core/serializers/python.py index 2929874b0179..53a73e19e51f 100644 --- a/django/core/serializers/python.py +++ b/django/core/serializers/python.py @@ -34,7 +34,7 @@ def end_object(self, obj): def get_dump_object(self, obj): data = {"model": str(obj._meta)} - if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"): + if not self.use_natural_primary_keys or not self._resolve_natural_key(obj): data["pk"] = self._value_from_field(obj, obj._meta.pk) data["fields"] = self._current return data @@ -52,26 +52,25 @@ def handle_field(self, obj, field): self._current[field.name] = self._value_from_field(obj, field) def handle_fk_field(self, obj, field): - if self.use_natural_foreign_keys and hasattr( - field.remote_field.model, "natural_key" + if self.use_natural_foreign_keys and ( + natural_key_value := self._resolve_fk_natural_key(obj, field) ): - related = getattr(obj, field.name) - if related: - value = related.natural_key() - else: - value = None + value = natural_key_value else: value = self._value_from_field(obj, field) self._current[field.name] = value def handle_m2m_field(self, obj, field): if field.remote_field.through._meta.auto_created: - if self.use_natural_foreign_keys and hasattr( - field.remote_field.model, "natural_key" + if self.use_natural_foreign_keys and self._model_supports_natural_key( + field.remote_field.model ): def m2m_value(value): - return value.natural_key() + if natural := value.natural_key(): + return natural + else: + return self._value_from_field(value, value._meta.pk) def queryset_iterator(obj, field): attr = getattr(obj, field.name) diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index f8ec0865a76b..910c489a377b 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -73,7 +73,7 @@ def start_object(self, obj): self.indent(1) attrs = {"model": str(obj._meta)} - if not self.use_natural_primary_keys or not hasattr(obj, "natural_key"): + if not self.use_natural_primary_keys or not self._resolve_natural_key(obj): obj_pk = obj.pk if obj_pk is not None: attrs["pk"] = obj._meta.pk.value_to_string(obj) @@ -128,14 +128,11 @@ def handle_fk_field(self, obj, field): self._start_relational_field(field) related_att = getattr(obj, field.attname) if related_att is not None: - if self.use_natural_foreign_keys and hasattr( - field.remote_field.model, "natural_key" + if self.use_natural_foreign_keys and ( + natural_key_value := self._resolve_fk_natural_key(obj, field) ): - related = getattr(obj, field.name) - # If related object has a natural key, use it - related = related.natural_key() # Iterable natural keys are rolled out as subelements - for key_value in related: + for key_value in natural_key_value: self.xml.startElement("natural", {}) self.xml.characters(str(key_value)) self.xml.endElement("natural") @@ -153,19 +150,21 @@ def handle_m2m_field(self, obj, field): """ if field.remote_field.through._meta.auto_created: self._start_relational_field(field) - if self.use_natural_foreign_keys and hasattr( - field.remote_field.model, "natural_key" + if self.use_natural_foreign_keys and self._model_supports_natural_key( + field.remote_field.model ): # If the objects in the m2m have a natural key, use it def handle_m2m(value): - natural = value.natural_key() - # Iterable natural keys are rolled out as subelements - self.xml.startElement("object", {}) - for key_value in natural: - self.xml.startElement("natural", {}) - self.xml.characters(str(key_value)) - self.xml.endElement("natural") - self.xml.endElement("object") + if natural := self._resolve_natural_key(value): + # Iterable natural keys are rolled out as subelements + self.xml.startElement("object", {}) + for key_value in natural: + self.xml.startElement("natural", {}) + self.xml.characters(str(key_value)) + self.xml.endElement("natural") + self.xml.endElement("object") + else: + self.xml.addQuickElement("object", attrs={"pk": str(value.pk)}) def queryset_iterator(obj, field): attr = getattr(obj, field.name) diff --git a/docs/releases/6.1.txt b/docs/releases/6.1.txt index 1c533f1341fe..587fec10b309 100644 --- a/docs/releases/6.1.txt +++ b/docs/releases/6.1.txt @@ -281,7 +281,10 @@ Security Serialization ~~~~~~~~~~~~~ -* ... +* Subclasses of models defining the ``natural_key()`` method can now opt out of + natural key serialization by overriding the method to return an empty tuple: + ``()``. This ensures primary keys are serialized when using + :option:`dumpdata --natural-primary`. Signals ~~~~~~~ diff --git a/docs/topics/serialization.txt b/docs/topics/serialization.txt index 2b28f5e15a84..f801df05617d 100644 --- a/docs/topics/serialization.txt +++ b/docs/topics/serialization.txt @@ -638,6 +638,15 @@ command line flags to generate natural keys. natural keys during serialization, but *not* be able to load those key values, just don't define the ``get_by_natural_key()`` method. +Subclasses can opt out of natural key serialization by returning an empty tuple +(``()``) from ``natural_key()``. This tells the serializer to fall back to +the standard primary key. + +.. versionchanged:: 6.1 + + Support for opting out of natural key serialization by returning an empty + tuple was added. + .. _natural-keys-and-forward-references: Natural keys and forward references diff --git a/tests/serializers/models/natural.py b/tests/serializers/models/natural.py index cfc57b112785..f32d781a7618 100644 --- a/tests/serializers/models/natural.py +++ b/tests/serializers/models/natural.py @@ -2,6 +2,7 @@ import uuid +from django.contrib.auth.base_user import AbstractBaseUser from django.db import models @@ -74,3 +75,19 @@ class FKAsPKNoNaturalKey(models.Model): def natural_key(self): raise NotImplementedError("This method was not expected to be called.") + + +class SubclassNaturalKeyOptOutUser(AbstractBaseUser): + email = models.EmailField(unique=False, null=True, blank=True) + USERNAME_FIELD = "email" + + def natural_key(self): + return () + + +class PostToOptOutSubclassUser(models.Model): + author = models.ForeignKey(SubclassNaturalKeyOptOutUser, on_delete=models.CASCADE) + title = models.CharField(max_length=100) + subscribers = models.ManyToManyField( + SubclassNaturalKeyOptOutUser, related_name="subscribed_posts", blank=True + ) diff --git a/tests/serializers/test_natural.py b/tests/serializers/test_natural.py index b5b35708c66c..322abeb7fc60 100644 --- a/tests/serializers/test_natural.py +++ b/tests/serializers/test_natural.py @@ -9,6 +9,8 @@ NaturalKeyAnchor, NaturalKeyThing, NaturalPKWithDefault, + PostToOptOutSubclassUser, + SubclassNaturalKeyOptOutUser, ) from .tests import register_tests @@ -250,6 +252,34 @@ def fk_as_pk_natural_key_not_called(self, format): self.assertEqual(obj.object.pk, o1.pk) +def natural_key_opt_out_test(self, format): + """ + When a subclass of AbstractBaseUser opts out of natural key serialization + by returning an empty tuple, both FK and M2M relations serialize as + integer PKs and can be deserialized without error. + """ + user1 = SubclassNaturalKeyOptOutUser.objects.create(email="user1@example.com") + user2 = SubclassNaturalKeyOptOutUser.objects.create(email="user2@example.com") + + post = PostToOptOutSubclassUser.objects.create( + author=user1, title="Post 2 (Subclass Opt-out)" + ) + post.subscribers.add(user1, user2) + + user_data = serializers.serialize(format, [user1], use_natural_primary_keys=True) + post_data = serializers.serialize(format, [post], use_natural_foreign_keys=True) + + list(serializers.deserialize(format, user_data)) + deserialized_posts = list(serializers.deserialize(format, post_data)) + + post_obj = deserialized_posts[0].object + self.assertEqual(user1.email, post_obj.author.email) + self.assertEqual( + sorted([user1.email, user2.email]), + sorted(post_obj.subscribers.values_list("email", flat=True)), + ) + + # Dynamically register tests for each serializer register_tests( NaturalKeySerializerTests, @@ -284,3 +314,8 @@ def fk_as_pk_natural_key_not_called(self, format): "test_%s_fk_as_pk_natural_key_not_called", fk_as_pk_natural_key_not_called, ) +register_tests( + NaturalKeySerializerTests, + "test_%s_natural_key_opt_out", + natural_key_opt_out_test, +) From bd4a562a8849147d4aa4bd42f7fdb1b51f89bb84 Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Wed, 26 Nov 2025 10:00:38 -0500 Subject: [PATCH 3/3] Closed pool when parallel test runner encounters unpicklable exceptions. --- django/test/runner.py | 59 +++++++++++++++--------------- tests/test_runner/test_parallel.py | 5 +-- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/django/test/runner.py b/django/test/runner.py index ecae164d7fce..5cd72119f5aa 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -567,7 +567,14 @@ def run(self, result): """ self.initialize_suite() counter = multiprocessing.Value(ctypes.c_int, 0) - pool = multiprocessing.Pool( + args = [ + (self.runner_class, index, subsuite, self.failfast, self.buffer) + for index, subsuite in enumerate(self.subsuites) + ] + # Don't buffer in the main process to avoid error propagation issues. + result.buffer = False + + with multiprocessing.Pool( processes=self.processes, initializer=functools.partial(_safe_init_worker, self.init_worker.__func__), initargs=[ @@ -579,38 +586,30 @@ def run(self, result): self.debug_mode, self.used_aliases, ], - ) - args = [ - (self.runner_class, index, subsuite, self.failfast, self.buffer) - for index, subsuite in enumerate(self.subsuites) - ] - # Don't buffer in the main process to avoid error propagation issues. - result.buffer = False - - test_results = pool.imap_unordered(self.run_subsuite.__func__, args) - - while True: - if result.shouldStop: - pool.terminate() - break - - try: - subsuite_index, events = test_results.next(timeout=0.1) - except multiprocessing.TimeoutError as err: - if counter.value < 0: - err.add_note("ERROR: _init_worker failed, see prior traceback") + ) as pool: + test_results = pool.imap_unordered(self.run_subsuite.__func__, args) + + while True: + if result.shouldStop: + pool.terminate() + break + + try: + subsuite_index, events = test_results.next(timeout=0.1) + except multiprocessing.TimeoutError as err: + if counter.value < 0: + err.add_note("ERROR: _init_worker failed, see prior traceback") + raise + continue + except StopIteration: pool.close() - raise - continue - except StopIteration: - pool.close() - break + break - tests = list(self.subsuites[subsuite_index]) - for event in events: - self.handle_event(result, tests, event) + tests = list(self.subsuites[subsuite_index]) + for event in events: + self.handle_event(result, tests, event) - pool.join() + pool.join() return result diff --git a/tests/test_runner/test_parallel.py b/tests/test_runner/test_parallel.py index 32cc971d305d..193afea1cc9e 100644 --- a/tests/test_runner/test_parallel.py +++ b/tests/test_runner/test_parallel.py @@ -309,9 +309,8 @@ def fake_next(*args, **kwargs): test_result.shouldStop = True return (0, remote_result.events) - mock_pool.return_value.imap_unordered.return_value = unittest.mock.Mock( - next=fake_next - ) + mock_imap = mock_pool.return_value.__enter__.return_value.imap_unordered + mock_imap.return_value = unittest.mock.Mock(next=fake_next) pts.run(test_result) self.assertIn("ValueError: woops", test_result.errors[0][1])