Fixed #22288 -- Fixed F() expressions with the __range lookup.

tests/expressions/models.py

@@ -61,6 +61,15 @@ class Experiment(models.Model):
         return self.end - self.start
 
 
+@python_2_unicode_compatible
+class Result(models.Model):
+    experiment = models.ForeignKey(Experiment, models.CASCADE)
+    result_time = models.DateTimeField()
+
+    def __str__(self):
+        return "Result at %s" % self.result_time
+
+
 @python_2_unicode_compatible
 class Time(models.Model):
     time = models.TimeField(null=True)
@@ -69,6 +78,16 @@ class Time(models.Model):
         return "%s" % self.time
 
 
+@python_2_unicode_compatible
+class SimulationRun(models.Model):
+    start = models.ForeignKey(Time, models.CASCADE, null=True)
+    end = models.ForeignKey(Time, models.CASCADE, null=True)
+    midpoint = models.TimeField()
+
+    def __str__(self):
+        return "%s (%s to %s)" % (self.midpoint, self.start, self.end)
+
+
 @python_2_unicode_compatible
 class UUID(models.Model):
     uuid = models.UUIDField(null=True)

tests/expressions/tests.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import datetime
+import unittest
 import uuid
 from copy import deepcopy
 
@@ -17,11 +18,15 @@ from django.db.models.expressions import (
 from django.db.models.functions import (
     Coalesce, Concat, Length, Lower, Substr, Upper,
 )
+from django.db.models.sql import constants
+from django.db.models.sql.datastructures import Join
 from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature
 from django.test.utils import Approximate
 from django.utils import six
 
-from .models import UUID, Company, Employee, Experiment, Number, Time
+from .models import (
+    UUID, Company, Employee, Experiment, Number, Result, SimulationRun, Time,
+)
 
 
 class BasicExpressionsTests(TestCase):
@@ -391,6 +396,144 @@ class BasicExpressionsTests(TestCase):
         self.assertEqual(str(qs.query).count('JOIN'), 2)
 
 
+class IterableLookupInnerExpressionsTests(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        ceo = Employee.objects.create(firstname='Just', lastname='Doit', salary=30)
+        # MySQL requires that the values calculated for expressions don't pass
+        # outside of the field's range, so it's inconvenient to use the values
+        # in the more general tests.
+        Company.objects.create(name='5020 Ltd', num_employees=50, num_chairs=20, ceo=ceo)
+        Company.objects.create(name='5040 Ltd', num_employees=50, num_chairs=40, ceo=ceo)
+        Company.objects.create(name='5050 Ltd', num_employees=50, num_chairs=50, ceo=ceo)
+        Company.objects.create(name='5060 Ltd', num_employees=50, num_chairs=60, ceo=ceo)
+        Company.objects.create(name='99300 Ltd', num_employees=99, num_chairs=300, ceo=ceo)
+
+    def test_in_lookup_allows_F_expressions_and_expressions_for_integers(self):
+        # __in lookups can use F() expressions for integers.
+        queryset = Company.objects.filter(num_employees__in=([F('num_chairs') - 10]))
+        self.assertQuerysetEqual(queryset, ['<Company: 5060 Ltd>'], ordered=False)
+        self.assertQuerysetEqual(
+            Company.objects.filter(num_employees__in=([F('num_chairs') - 10, F('num_chairs') + 10])),
+            ['<Company: 5040 Ltd>', '<Company: 5060 Ltd>'],
+            ordered=False
+        )
+        self.assertQuerysetEqual(
+            Company.objects.filter(
+                num_employees__in=([F('num_chairs') - 10, F('num_chairs'), F('num_chairs') + 10])
+            ),
+            ['<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
+            ordered=False
+        )
+
+    def test_expressions_in_lookups_join_choice(self):
+        midpoint = datetime.time(13, 0)
+        t1 = Time.objects.create(time=datetime.time(12, 0))
+        t2 = Time.objects.create(time=datetime.time(14, 0))
+        SimulationRun.objects.create(start=t1, end=t2, midpoint=midpoint)
+        SimulationRun.objects.create(start=t1, end=None, midpoint=midpoint)
+        SimulationRun.objects.create(start=None, end=t2, midpoint=midpoint)
+        SimulationRun.objects.create(start=None, end=None, midpoint=midpoint)
+
+        queryset = SimulationRun.objects.filter(midpoint__range=[F('start__time'), F('end__time')])
+        self.assertQuerysetEqual(
+            queryset,
+            ['<SimulationRun: 13:00:00 (12:00:00 to 14:00:00)>'],
+            ordered=False
+        )
+        for alias in queryset.query.alias_map.values():
+            if isinstance(alias, Join):
+                self.assertEqual(alias.join_type, constants.INNER)
+
+        queryset = SimulationRun.objects.exclude(midpoint__range=[F('start__time'), F('end__time')])
+        self.assertQuerysetEqual(queryset, [], ordered=False)
+        for alias in queryset.query.alias_map.values():
+            if isinstance(alias, Join):
+                self.assertEqual(alias.join_type, constants.LOUTER)
+
+    def test_range_lookup_allows_F_expressions_and_expressions_for_integers(self):
+        # Range lookups can use F() expressions for integers.
+        Company.objects.filter(num_employees__exact=F("num_chairs"))
+        self.assertQuerysetEqual(
+            Company.objects.filter(num_employees__range=(F('num_chairs'), 100)),
+            ['<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>'],
+            ordered=False
+        )
+        self.assertQuerysetEqual(
+            Company.objects.filter(num_employees__range=(F('num_chairs') - 10, F('num_chairs') + 10)),
+            ['<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
+            ordered=False
+        )
+        self.assertQuerysetEqual(
+            Company.objects.filter(num_employees__range=(F('num_chairs') - 10, 100)),
+            ['<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>', '<Company: 5060 Ltd>'],
+            ordered=False
+        )
+        self.assertQuerysetEqual(
+            Company.objects.filter(num_employees__range=(1, 100)),
+            [
+                '<Company: 5020 Ltd>', '<Company: 5040 Ltd>', '<Company: 5050 Ltd>',
+                '<Company: 5060 Ltd>', '<Company: 99300 Ltd>',
+            ],
+            ordered=False
+        )
+
+    @unittest.skipUnless(connection.vendor == 'sqlite',
+                         "This defensive test only works on databases that don't validate parameter types")
+    def test_complex_expressions_do_not_introduce_sql_injection_via_untrusted_string_inclusion(self):
+        """
+        This tests that SQL injection isn't possible using compilation of
+        expressions in iterable filters, as their compilation happens before
+        the main query compilation. It's limited to SQLite, as PostgreSQL,
+        Oracle and other vendors have defense in depth against this by type
+        checking. Testing against SQLite (the most permissive of the built-in
+        databases) demonstrates that the problem doesn't exist while keeping
+        the test simple.
+        """
+        queryset = Company.objects.filter(name__in=[F('num_chairs') + '1)) OR ((1==1'])
+        self.assertQuerysetEqual(queryset, [], ordered=False)
+
+    def test_in_lookup_allows_F_expressions_and_expressions_for_datetimes(self):
+        start = datetime.datetime(2016, 2, 3, 15, 0, 0)
+        end = datetime.datetime(2016, 2, 5, 15, 0, 0)
+        experiment_1 = Experiment.objects.create(
+            name='Integrity testing',
+            assigned=start.date(),
+            start=start,
+            end=end,
+            completed=end.date(),
+            estimated_time=end - start,
+        )
+        experiment_2 = Experiment.objects.create(
+            name='Taste testing',
+            assigned=start.date(),
+            start=start,
+            end=end,
+            completed=end.date(),
+            estimated_time=end - start,
+        )
+        Result.objects.create(
+            experiment=experiment_1,
+            result_time=datetime.datetime(2016, 2, 4, 15, 0, 0),
+        )
+        Result.objects.create(
+            experiment=experiment_1,
+            result_time=datetime.datetime(2016, 3, 10, 2, 0, 0),
+        )
+        Result.objects.create(
+            experiment=experiment_2,
+            result_time=datetime.datetime(2016, 1, 8, 5, 0, 0),
+        )
+
+        within_experiment_time = [F('experiment__start'), F('experiment__end')]
+        queryset = Result.objects.filter(result_time__range=within_experiment_time)
+        self.assertQuerysetEqual(queryset, ["<Result: Result at 2016-02-04 15:00:00>"])
+
+        within_experiment_time = [F('experiment__start'), F('experiment__end')]
+        queryset = Result.objects.filter(result_time__range=within_experiment_time)
+        self.assertQuerysetEqual(queryset, ["<Result: Result at 2016-02-04 15:00:00>"])
+
+
 class ExpressionsTests(TestCase):
 
     def test_F_object_deepcopy(self):

tests/postgres_tests/test_array.py

@@ -173,12 +173,40 @@ class TestQuerying(PostgreSQLTestCase):
             self.objs[:2]
         )
 
+    @unittest.expectedFailure
+    def test_in_including_F_object(self):
+        # This test asserts that Array objects passed to filters can be
+        # constructed to contain F objects. This currently doesn't work as the
+        # psycopg2 mogrify method that generates the ARRAY() syntax is
+        # expecting literals, not column references (#27095).
+        self.assertSequenceEqual(
+            NullableIntegerArrayModel.objects.filter(field__in=[[models.F('id')]]),
+            self.objs[:2]
+        )
+
+    def test_in_as_F_object(self):
+        self.assertSequenceEqual(
+            NullableIntegerArrayModel.objects.filter(field__in=[models.F('field')]),
+            self.objs[:4]
+        )
+
     def test_contained_by(self):
         self.assertSequenceEqual(
             NullableIntegerArrayModel.objects.filter(field__contained_by=[1, 2]),
             self.objs[:2]
         )
 
+    @unittest.expectedFailure
+    def test_contained_by_including_F_object(self):
+        # This test asserts that Array objects passed to filters can be
+        # constructed to contain F objects. This currently doesn't work as the
+        # psycopg2 mogrify method that generates the ARRAY() syntax is
+        # expecting literals, not column references (#27095).
+        self.assertSequenceEqual(
+            NullableIntegerArrayModel.objects.filter(field__contained_by=[models.F('id'), 2]),
+            self.objs[:2]
+        )
+
     def test_contains(self):
         self.assertSequenceEqual(
             NullableIntegerArrayModel.objects.filter(field__contains=[2]),