Django 2.0 compatibility (closes #1582)

Thanks to @mpauly and @timgraham for working on this and @dani0805,
@andrewbenedictwallace, @RabidCicada, @webtweakers, @nadimtuhin, and
@JonLevischi for testing.

Author: acdha

.travis.yml

@@ -5,6 +5,7 @@ python:
     - 2.7
     - 3.4
     - 3.5
+    - 3.6
     - pypy
 
 cache:
@@ -58,17 +59,19 @@ script:
 
 env:
     matrix:
-        - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.11,<1.12" VERSION_ES=">=2.0.0,<3.0.0"
+        - DJANGO_VERSION=">=1.11,<2.0" VERSION_ES=">=1.0.0,<2.0.0"
+        - DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=1.0.0,<2.0.0"
+        - DJANGO_VERSION=">=1.11,<2.0" VERSION_ES=">=2.0.0,<3.0.0"
+        - DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=2.0.0,<3.0.0"
 
 matrix:
     allow_failures:
         - python: 'pypy'
+    exclude:
+        - python: 2.7
+          env: DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=2.0.0,<3.0.0"
+        - python: 2.7
+          env: DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=1.0.0,<2.0.0"
 
 notifications:
     irc: "irc.freenode.org#haystack"

AUTHORS

@@ -117,3 +117,4 @@ Thanks to
     * Morgan Aubert (@ellmetha) for Django 1.10 support
     * João Junior (@joaojunior) and Bruno Marques (@ElSaico) for Elasticsearch 2.x support
     * Alex Tomkins (@tomkins) for various patches
+    * Martin Pauly (@mpauly) for Django 2.0 support

haystack/backends/elasticsearch_backend.py

@@ -296,7 +296,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
             for field, direction in sort_by:
                 if field == 'distance' and distance_point:
                     # Do the geo-enabled sort.
-                    lng, lat = distance_point['point'].get_coords()
+                    lng, lat = distance_point['point'].coords
                     sort_kwargs = {
                         "_geo_distance": {
                             distance_point['field']: [lng, lat],
@@ -455,7 +455,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
             filters.append(within_filter)
 
         if dwithin is not None:
-            lng, lat = dwithin['point'].get_coords()
+            lng, lat = dwithin['point'].coords
 
             # NB: the 1.0.0 release of elasticsearch introduce an
             #     incompatible change on the distance filter formating

haystack/backends/solr_backend.py

@@ -173,7 +173,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
         if sort_by is not None:
             if sort_by in ['distance asc', 'distance desc'] and distance_point:
                 # Do the geo-enabled sort.
-                lng, lat = distance_point['point'].get_coords()
+                lng, lat = distance_point['point'].coords
                 kwargs['sfield'] = distance_point['field']
                 kwargs['pt'] = '%s,%s' % (lat, lng)
 
@@ -290,7 +290,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
 
         if dwithin is not None:
             kwargs.setdefault('fq', [])
-            lng, lat = dwithin['point'].get_coords()
+            lng, lat = dwithin['point'].coords
             geofilt = '{!geofilt pt=%s,%s sfield=%s d=%s}' % (lat, lng, dwithin['field'], dwithin['distance'].km)
             kwargs['fq'].append(geofilt)
 

haystack/indexes.py

@@ -443,7 +443,7 @@ def should_skip_field(self, field):
             return True
 
         # Ignore certain fields (AutoField, related fields).
-        if field.primary_key or getattr(field, 'rel'):
+        if field.primary_key or field.is_relation:
             return True
 
         return False

haystack/management/commands/rebuild_index.py

@@ -1,5 +1,4 @@
 # encoding: utf-8
-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from django.core.management import call_command
@@ -33,5 +32,11 @@ def add_arguments(self, parser):
         )
 
     def handle(self, **options):
-        call_command('clear_index', **options)
-        call_command('update_index', **options)
+        clear_options = options.copy()
+        update_options = options.copy()
+        for key in ('batchsize', 'workers'):
+            del clear_options[key]
+        for key in ('interactive', ):
+            del update_options[key]
+        call_command('clear_index', **clear_options)
+        call_command('update_index', **update_options)

haystack/models.py

@@ -133,7 +133,7 @@ def _get_distance(self):
             if location_field is None:
                 return None
 
-            lf_lng, lf_lat = location_field.get_coords()
+            lf_lng, lf_lat = location_field.coords
             self._distance = Distance(km=geopy_distance.distance((po_lat, po_lng), (lf_lat, lf_lng)).km)
 
         # We've either already calculated it or the backend returned it, so

haystack/utils/geo.py

@@ -43,7 +43,7 @@ def ensure_wgs84(point):
 
     if not new_point.srid:
         # It has no spatial reference id. Assume WGS-84.
-        new_point.set_srid(WGS_84_SRID)
+        new_point.srid = WGS_84_SRID
     elif new_point.srid != WGS_84_SRID:
         # Transform it to get to the right system.
         new_point.transform(WGS_84_SRID)
@@ -72,7 +72,7 @@ def generate_bounding_box(bottom_left, top_right):
 
     The two-tuple is in the form ``((min_lat, min_lng), (max_lat, max_lng))``.
     """
-    west, lat_1 = bottom_left.get_coords()
-    east, lat_2 = top_right.get_coords()
+    west, lat_1 = bottom_left.coords
+    east, lat_2 = top_right.coords
     min_lat, max_lat = min(lat_1, lat_2), max(lat_1, lat_2)
     return ((min_lat, west), (max_lat, east))

setup.py

@@ -12,7 +12,7 @@
     from setuptools import setup
 
 install_requires = [
-    'Django>=1.8,<1.12',
+    'Django>=1.11',
 ]
 
 tests_require = [
@@ -54,6 +54,8 @@
         'Development Status :: 5 - Production/Stable',
         'Environment :: Web Environment',
         'Framework :: Django',
+        'Framework :: Django :: 1.11',
+        'Framework :: Django :: 2.0',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',

test_haystack/core/models.py

@@ -17,7 +17,7 @@ class MockModel(models.Model):
     author = models.CharField(max_length=255)
     foo = models.CharField(max_length=255, blank=True)
     pub_date = models.DateTimeField(default=datetime.datetime.now)
-    tag = models.ForeignKey(MockTag)
+    tag = models.ForeignKey(MockTag, models.CASCADE)
 
     def __unicode__(self):
         return self.author
@@ -108,4 +108,4 @@ class OneToManyLeftSideModel(models.Model):
 
 
 class OneToManyRightSideModel(models.Model):
-    left_side = models.ForeignKey(OneToManyLeftSideModel, related_name='right_side')
+    left_side = models.ForeignKey(OneToManyLeftSideModel, models.CASCADE, related_name='right_side')

test_haystack/core/urls.py

@@ -13,7 +13,7 @@
 
 
 urlpatterns = [
-    url(r'^admin/', include(admin.site.urls)),
+    url(r'^admin/', admin.site.urls),
 
     url(r'^$', SearchView(load_all=False), name='haystack_search'),
     url(r'^faceted/$',
@@ -23,5 +23,5 @@
 ]
 
 urlpatterns += [
-    url(r'', include('test_haystack.test_app_without_models.urls', namespace='app-without-models')),
+    url(r'', include(('test_haystack.test_app_without_models.urls', 'app-without-models'))),
 ]

test_haystack/settings.py

@@ -51,12 +51,11 @@
     },
 ]
 
-MIDDLEWARE_CLASSES = [
+MIDDLEWARE = [
     'django.middleware.common.CommonMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
 ]
 

test_haystack/solr_tests/test_admin.py

@@ -6,7 +6,7 @@
 from django.contrib.auth.models import User
 from django.test import TestCase
 from django.test.utils import override_settings
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 from haystack import connections, reset_search_queries
 from haystack.utils.loading import UnifiedIndex

test_haystack/solr_tests/test_solr_management_commands.py

@@ -194,7 +194,7 @@ def test_build_schema_wrong_backend(self):
                                                    'PATH': mkdtemp(prefix='dummy-path-'), }
 
         connections['whoosh']._index = self.ui
-        self.assertRaises(ImproperlyConfigured, call_command, 'build_solr_schema', using='whoosh', interactive=False)
+        self.assertRaises(ImproperlyConfigured, call_command, 'build_solr_schema', using='whoosh')
 
     def test_build_schema(self):
 
@@ -276,38 +276,38 @@ def test_app_model_variations(self):
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
-        call_command('update_index', 'core', interactive=False, verbosity=0)
+        call_command('update_index', 'core', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 25)
 
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
         with self.assertRaises(ImproperlyConfigured):
-            call_command('update_index', 'fake_app_thats_not_there', interactive=False)
+            call_command('update_index', 'fake_app_thats_not_there')
 
-        call_command('update_index', 'core', 'discovery', interactive=False, verbosity=0)
+        call_command('update_index', 'core', 'discovery', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 25)
 
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
-        call_command('update_index', 'discovery', interactive=False, verbosity=0)
+        call_command('update_index', 'discovery', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
-        call_command('update_index', 'core.MockModel', interactive=False, verbosity=0)
+        call_command('update_index', 'core.MockModel', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 23)
 
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
-        call_command('update_index', 'core.MockTag', interactive=False, verbosity=0)
+        call_command('update_index', 'core.MockTag', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 2)
 
         call_command('clear_index', interactive=False, verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 0)
 
-        call_command('update_index', 'core.MockTag', 'core.MockModel', interactive=False, verbosity=0)
+        call_command('update_index', 'core.MockTag', 'core.MockModel', verbosity=0)
         self.assertEqual(self.solr.search('*:*').hits, 25)

test_haystack/spatial/test_spatial.py

@@ -37,7 +37,7 @@ def test_ensure_wgs84(self):
         self.assertEqual(std_pnt.y, 38.97127105172941)
 
         orig_pnt = Point(-95.23592948913574, 38.97127105172941)
-        orig_pnt.set_srid(2805)
+        orig_pnt.srid = 2805
         std_pnt = ensure_wgs84(orig_pnt)
         self.assertEqual(orig_pnt.srid, 2805)
         self.assertEqual(std_pnt.srid, 4326)
@@ -96,8 +96,8 @@ def test_indexing(self):
         self.assertEqual(sqs.count(), 1)
         self.assertEqual(sqs[0].username, first.username)
         # Make sure we've got a proper ``Point`` object.
-        self.assertAlmostEqual(sqs[0].location.get_coords()[0], first.longitude)
-        self.assertAlmostEqual(sqs[0].location.get_coords()[1], first.latitude)
+        self.assertAlmostEqual(sqs[0].location.coords[0], first.longitude)
+        self.assertAlmostEqual(sqs[0].location.coords[1], first.latitude)
 
         # Double-check, to make sure there was nothing accidentally copied
         # between instances.
@@ -106,8 +106,8 @@ def test_indexing(self):
         sqs = self.sqs.models(Checkin).filter(django_id=second.pk)
         self.assertEqual(sqs.count(), 1)
         self.assertEqual(sqs[0].username, second.username)
-        self.assertAlmostEqual(sqs[0].location.get_coords()[0], second.longitude)
-        self.assertAlmostEqual(sqs[0].location.get_coords()[1], second.latitude)
+        self.assertAlmostEqual(sqs[0].location.coords[0], second.longitude)
+        self.assertAlmostEqual(sqs[0].location.coords[1], second.latitude)
 
     def test_within(self):
         self.assertEqual(self.sqs.all().count(), 10)

test_haystack/test_app_loading.py

@@ -3,7 +3,7 @@
 
 from types import GeneratorType, ModuleType
 
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.test import TestCase
 
 from haystack.utils import app_loading

test_haystack/test_indexes.py

@@ -581,6 +581,11 @@ def read_queryset(self, using=None):
         return self.get_model().objects.complete_set()
 
 
+class ModelWithManyToManyFieldModelSearchIndex(indexes.ModelSearchIndex):
+    def get_model(self):
+        return ManyToManyLeftSideModel
+
+
 class ModelSearchIndexTestCase(TestCase):
     def setUp(self):
         super(ModelSearchIndexTestCase, self).setUp()
@@ -590,6 +595,7 @@ def setUp(self):
         self.emsi = ExcludesModelSearchIndex()
         self.fwomsi = FieldsWithOverrideModelSearchIndex()
         self.yabmsi = YetAnotherBasicModelSearchIndex()
+        self.m2mmsi = ModelWithManyToManyFieldModelSearchIndex()
 
     def test_basic(self):
         self.assertEqual(len(self.bmsi.fields), 4)
@@ -623,6 +629,7 @@ def test_excludes(self):
         self.assertTrue(isinstance(self.emsi.fields['pub_date'], indexes.DateTimeField))
         self.assertTrue('text' in self.emsi.fields)
         self.assertTrue(isinstance(self.emsi.fields['text'], indexes.CharField))
+        self.assertNotIn('related_models', self.m2mmsi.fields)
 
     def test_fields_with_override(self):
         self.assertEqual(len(self.fwomsi.fields), 3)

test_haystack/test_management_commands.py

@@ -66,8 +66,8 @@ def test_rebuild_index_using(self, m1, m2):
         m2.assert_called_with("eng")
         m1.assert_any_call("core", "eng")
 
-    @patch('haystack.management.commands.update_index.Command.handle')
-    @patch('haystack.management.commands.clear_index.Command.handle')
+    @patch('haystack.management.commands.update_index.Command.handle', return_value='')
+    @patch('haystack.management.commands.clear_index.Command.handle', return_value='')
     def test_rebuild_index(self, mock_handle_clear, mock_handle_update):
         call_command('rebuild_index', interactive=False)
 
@@ -87,9 +87,9 @@ def test_rebuild_index_nocommit(self, *mocks):
             self.assertIn('commit', kwargs)
             self.assertEqual(False, kwargs['commit'])
 
-    @patch('haystack.management.commands.update_index.Command.handle')
-    @patch('haystack.management.commands.clear_index.Command.handle')
-    def test_rebuild_index_nocommit(self, *mocks):
+    @patch('haystack.management.commands.clear_index.Command.handle', return_value='')
+    @patch('haystack.management.commands.update_index.Command.handle', return_value='')
+    def test_rebuild_index_nocommit(self, update_mock, clear_mock):
         """
         Confirm that command-line option parsing produces the same results as using call_command() directly,
         mostly as a sanity check for the logic in rebuild_index which combines the option_lists for its
@@ -99,13 +99,15 @@ def test_rebuild_index_nocommit(self, *mocks):
 
         Command().run_from_argv(['django-admin.py', 'rebuild_index', '--noinput', '--nocommit'])
 
-        for m in mocks:
+        for m in (clear_mock, update_mock):
             self.assertEqual(m.call_count, 1)
 
             args, kwargs = m.call_args
 
             self.assertIn('commit', kwargs)
             self.assertEqual(False, kwargs['commit'])
 
-            self.assertIn('interactive', kwargs)
-            self.assertEqual(False, kwargs['interactive'])
+        args, kwargs = clear_mock.call_args
+
+        self.assertIn('interactive', kwargs)
+        self.assertIs(kwargs['interactive'], False)

test_haystack/test_views.py

@@ -6,10 +6,10 @@
 from threading import Thread
 
 from django import forms
-from django.core.urlresolvers import reverse
 from django.http import HttpRequest, QueryDict
 from django.test import TestCase, override_settings
 from django.utils.six.moves import queue
+from django.urls import reverse
 from test_haystack.core.models import AnotherMockModel, MockModel
 
 from haystack import connections, indexes