Luke Plant avatar Luke Plant committed 22d8bd8

Fixed #17530 - Fixture loading with deferred constraints broken for Postgres for fixtures from multiple dirs/files

Thanks to claudep for the review.

Comments (0)

Files changed (4)

django/core/management/commands/loaddata.py

                 app_module_paths.append(app.__file__)
 
         app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
-        for fixture_label in fixture_labels:
-            parts = fixture_label.split('.')
 
-            if len(parts) > 1 and parts[-1] in compression_types:
-                compression_formats = [parts[-1]]
-                parts = parts[:-1]
-            else:
-                compression_formats = compression_types.keys()
+        try:
+            with connection.constraint_checks_disabled():
+                for fixture_label in fixture_labels:
+                    parts = fixture_label.split('.')
 
-            if len(parts) == 1:
-                fixture_name = parts[0]
-                formats = serializers.get_public_serializer_formats()
-            else:
-                fixture_name, format = '.'.join(parts[:-1]), parts[-1]
-                if format in serializers.get_public_serializer_formats():
-                    formats = [format]
-                else:
-                    formats = []
+                    if len(parts) > 1 and parts[-1] in compression_types:
+                        compression_formats = [parts[-1]]
+                        parts = parts[:-1]
+                    else:
+                        compression_formats = compression_types.keys()
 
-            if formats:
-                if verbosity >= 2:
-                    self.stdout.write("Loading '%s' fixtures...\n" % fixture_name)
-            else:
-                self.stderr.write(
-                    self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format.\n" %
-                        (fixture_name, format)))
-                if commit:
-                    transaction.rollback(using=using)
-                    transaction.leave_transaction_management(using=using)
-                return
+                    if len(parts) == 1:
+                        fixture_name = parts[0]
+                        formats = serializers.get_public_serializer_formats()
+                    else:
+                        fixture_name, format = '.'.join(parts[:-1]), parts[-1]
+                        if format in serializers.get_public_serializer_formats():
+                            formats = [format]
+                        else:
+                            formats = []
 
-            if os.path.isabs(fixture_name):
-                fixture_dirs = [fixture_name]
-            else:
-                fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
+                    if formats:
+                        if verbosity >= 2:
+                            self.stdout.write("Loading '%s' fixtures...\n" % fixture_name)
+                    else:
+                        self.stderr.write(
+                            self.style.ERROR("Problem installing fixture '%s': %s is not a known serialization format.\n" %
+                                (fixture_name, format)))
+                        if commit:
+                            transaction.rollback(using=using)
+                            transaction.leave_transaction_management(using=using)
+                        return
 
-            for fixture_dir in fixture_dirs:
-                if verbosity >= 2:
-                    self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir))
+                    if os.path.isabs(fixture_name):
+                        fixture_dirs = [fixture_name]
+                    else:
+                        fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + ['']
 
-                label_found = False
-                for combo in product([using, None], formats, compression_formats):
-                    database, format, compression_format = combo
-                    file_name = '.'.join(
-                        p for p in [
-                            fixture_name, database, format, compression_format
-                        ]
-                        if p
-                    )
+                    for fixture_dir in fixture_dirs:
+                        if verbosity >= 2:
+                            self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir))
 
-                    if verbosity >= 3:
-                        self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
-                            (humanize(fixture_dir), file_name, fixture_name))
-                    full_path = os.path.join(fixture_dir, file_name)
-                    open_method = compression_types[compression_format]
-                    try:
-                        fixture = open_method(full_path, 'r')
-                    except IOError:
-                        if verbosity >= 2:
-                            self.stdout.write("No %s fixture '%s' in %s.\n" % \
-                                (format, fixture_name, humanize(fixture_dir)))
-                    else:
-                        if label_found:
-                            fixture.close()
-                            self.stderr.write(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting.\n" %
-                                (fixture_name, humanize(fixture_dir))))
-                            if commit:
-                                transaction.rollback(using=using)
-                                transaction.leave_transaction_management(using=using)
-                            return
-                        else:
-                            fixture_count += 1
-                            objects_in_fixture = 0
-                            loaded_objects_in_fixture = 0
-                            if verbosity >= 2:
-                                self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
-                                    (format, fixture_name, humanize(fixture_dir)))
+                        label_found = False
+                        for combo in product([using, None], formats, compression_formats):
+                            database, format, compression_format = combo
+                            file_name = '.'.join(
+                                p for p in [
+                                    fixture_name, database, format, compression_format
+                                ]
+                                if p
+                            )
+
+                            if verbosity >= 3:
+                                self.stdout.write("Trying %s for %s fixture '%s'...\n" % \
+                                    (humanize(fixture_dir), file_name, fixture_name))
+                            full_path = os.path.join(fixture_dir, file_name)
+                            open_method = compression_types[compression_format]
                             try:
-                                objects = serializers.deserialize(format, fixture, using=using)
+                                fixture = open_method(full_path, 'r')
+                            except IOError:
+                                if verbosity >= 2:
+                                    self.stdout.write("No %s fixture '%s' in %s.\n" % \
+                                        (format, fixture_name, humanize(fixture_dir)))
+                            else:
+                                try:
+                                    if label_found:
+                                        self.stderr.write(self.style.ERROR("Multiple fixtures named '%s' in %s. Aborting.\n" %
+                                            (fixture_name, humanize(fixture_dir))))
+                                        if commit:
+                                            transaction.rollback(using=using)
+                                            transaction.leave_transaction_management(using=using)
+                                        return
 
-                                with connection.constraint_checks_disabled():
+                                    fixture_count += 1
+                                    objects_in_fixture = 0
+                                    loaded_objects_in_fixture = 0
+                                    if verbosity >= 2:
+                                        self.stdout.write("Installing %s fixture '%s' from %s.\n" % \
+                                            (format, fixture_name, humanize(fixture_dir)))
+
+                                    objects = serializers.deserialize(format, fixture, using=using)
+
                                     for obj in objects:
                                         objects_in_fixture += 1
                                         if router.allow_syncdb(using, obj.object.__class__):
                                                     }
                                                 raise e.__class__, e.__class__(msg), sys.exc_info()[2]
 
-                                # Since we disabled constraint checks, we must manually check for
-                                # any invalid keys that might have been added
-                                table_names = [model._meta.db_table for model in models]
-                                connection.check_constraints(table_names=table_names)
+                                    loaded_object_count += loaded_objects_in_fixture
+                                    fixture_object_count += objects_in_fixture
+                                    label_found = True
+                                finally:
+                                    fixture.close()
 
-                                loaded_object_count += loaded_objects_in_fixture
-                                fixture_object_count += objects_in_fixture
-                                label_found = True
-                            except (SystemExit, KeyboardInterrupt):
-                                raise
-                            except Exception:
-                                fixture.close()
-                                if commit:
-                                    transaction.rollback(using=using)
-                                    transaction.leave_transaction_management(using=using)
-                                if show_traceback:
-                                    traceback.print_exc()
-                                else:
+                                # If the fixture we loaded contains 0 objects, assume that an
+                                # error was encountered during fixture loading.
+                                if objects_in_fixture == 0:
                                     self.stderr.write(
-                                        self.style.ERROR("Problem installing fixture '%s': %s\n" %
-                                             (full_path, ''.join(traceback.format_exception(sys.exc_type,
-                                                 sys.exc_value, sys.exc_traceback)))))
-                                return
-                            fixture.close()
+                                        self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)\n" %
+                                            (fixture_name)))
+                                    if commit:
+                                        transaction.rollback(using=using)
+                                        transaction.leave_transaction_management(using=using)
+                                    return
 
-                            # If the fixture we loaded contains 0 objects, assume that an
-                            # error was encountered during fixture loading.
-                            if objects_in_fixture == 0:
-                                self.stderr.write(
-                                    self.style.ERROR("No fixture data found for '%s'. (File format may be invalid.)\n" %
-                                        (fixture_name)))
-                                if commit:
-                                    transaction.rollback(using=using)
-                                    transaction.leave_transaction_management(using=using)
-                                return
+            # Since we disabled constraint checks, we must manually check for
+            # any invalid keys that might have been added
+            table_names = [model._meta.db_table for model in models]
+            connection.check_constraints(table_names=table_names)
+
+        except (SystemExit, KeyboardInterrupt):
+            raise
+        except Exception:
+            if commit:
+                transaction.rollback(using=using)
+                transaction.leave_transaction_management(using=using)
+            if show_traceback:
+                traceback.print_exc()
+            else:
+                self.stderr.write(
+                    self.style.ERROR("Problem installing fixture '%s': %s\n" %
+                         (full_path, ''.join(traceback.format_exception(sys.exc_type,
+                             sys.exc_value, sys.exc_traceback)))))
+            return
+
 
         # If we found even one object in a fixture, we need to reset the
         # database sequences.

tests/regressiontests/fixtures_regress/fixtures_1/forward_ref_1.json

+[
+    {
+        "pk": 1,
+        "model": "fixtures_regress.book",
+        "fields": {
+            "name": "Cryptonomicon",
+            "author": 4
+        }
+    }
+]

tests/regressiontests/fixtures_regress/fixtures_2/forward_ref_2.json

+[
+    {
+        "pk": "4",
+        "model": "fixtures_regress.person",
+        "fields": {
+            "name": "Neal Stephenson"
+        }
+    }
+]

tests/regressiontests/fixtures_regress/tests.py

 from django.db.models import signals
 from django.test import (TestCase, TransactionTestCase, skipIfDBFeature,
     skipUnlessDBFeature)
+from django.test.utils import override_settings
 
 from .models import (Animal, Stuff, Absolute, Parent, Child, Article, Widget,
     Store, Person, Book, NKChild, RefToNKChild, Circle1, Circle2, Circle3,
             stderr.getvalue().startswith('Problem installing fixture')
         )
 
+    _cur_dir = os.path.dirname(os.path.abspath(__file__))
+
+    @override_settings(FIXTURE_DIRS=[os.path.join(_cur_dir, 'fixtures_1'),
+                                     os.path.join(_cur_dir, 'fixtures_2')])
+    def test_loaddata_forward_refs_split_fixtures(self):
+        """
+        Regression for #17530 - should be able to cope with forward references
+        when the fixtures are not in the same files or directories.
+        """
+        management.call_command(
+            'loaddata',
+            'forward_ref_1.json',
+            'forward_ref_2.json',
+            verbosity=0,
+            commit=False
+        )
+        self.assertEqual(Book.objects.all()[0].id, 1)
+        self.assertEqual(Person.objects.all()[0].id, 4)
+
     def test_loaddata_no_fixture_specified(self):
         """
         Regression for #7043 - Error is quickly reported when no fixtures is provided in the command line.
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.