From a2e927b7ed25bd319f866c55b18cd214a149d859 Mon Sep 17 00:00:00 2001 From: Mike Grouchy Date: Wed, 18 Jul 2012 08:00:31 -0400 Subject: [PATCH 001/302] BaseCache now has a no-op close method as per ticket #18582 Also removed the hasattr check when firing request_finished signal for caches with a 'close' method. Should be safe to call `cache.close` everywhere now --- AUTHORS | 1 + django/core/cache/__init__.py | 8 +++----- django/core/cache/backends/base.py | 9 +++++++++ docs/topics/cache.txt | 10 ++++++++++ tests/regressiontests/cache/tests.py | 4 ++++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index ea0b01926c..44b6a90700 100644 --- a/AUTHORS +++ b/AUTHORS @@ -229,6 +229,7 @@ answer newbie questions, and generally made Django that much better: Simon Greenhill Owen Griffiths Espen Grindhaug + Mike Grouchy Janos Guljas Thomas Güttler Horst Gutmann diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py index 2a9e1a700b..457433925a 100644 --- a/django/core/cache/__init__.py +++ b/django/core/cache/__init__.py @@ -133,11 +133,9 @@ def get_cache(backend, **kwargs): "Could not find backend '%s': %s" % (backend, e)) cache = backend_cls(location, params) # Some caches -- python-memcached in particular -- need to do a cleanup at the - # end of a request cycle. If the cache provides a close() method, wire it up - # here. - if hasattr(cache, 'close'): - signals.request_finished.connect(cache.close) + # end of a request cycle. If not implemented in a particular backend + # cache.close is a no-op + signals.request_finished.connect(cache.close) return cache cache = get_cache(DEFAULT_CACHE_ALIAS) - diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index f7573b2e31..e28ebbe2cf 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -6,15 +6,18 @@ from django.core.exceptions import ImproperlyConfigured, DjangoRuntimeWarning from django.utils.encoding import smart_str from django.utils.importlib import import_module + class InvalidCacheBackendError(ImproperlyConfigured): pass + class CacheKeyWarning(DjangoRuntimeWarning): pass # Memcached does not accept keys longer than this. MEMCACHE_MAX_KEY_LENGTH = 250 + def default_key_func(key, key_prefix, version): """ Default function to generate keys. @@ -25,6 +28,7 @@ def default_key_func(key, key_prefix, version): """ return ':'.join([key_prefix, str(version), smart_str(key)]) + def get_key_func(key_func): """ Function to decide which key function to use. @@ -40,6 +44,7 @@ def get_key_func(key_func): return getattr(key_func_module, key_func_name) return default_key_func + class BaseCache(object): def __init__(self, params): timeout = params.get('timeout', params.get('TIMEOUT', 300)) @@ -221,3 +226,7 @@ class BaseCache(object): the new version. """ return self.incr_version(key, -delta, version) + + def close(self, **kwargs): + """Close the cache connection""" + pass diff --git a/docs/topics/cache.txt b/docs/topics/cache.txt index 03afa86647..85f4c64aa8 100644 --- a/docs/topics/cache.txt +++ b/docs/topics/cache.txt @@ -779,6 +779,16 @@ nonexistent cache key.:: However, if the backend doesn't natively provide an increment/decrement operation, it will be implemented using a two-step retrieve/update. + +You can close the connection to your cache with ``close()`` if implemented by +the cache backend. + + >>> cache.close() + +.. note:: + + For caches that don't implement ``close`` methods it is a no-op. + .. _cache_key_prefixing: Cache key prefixing diff --git a/tests/regressiontests/cache/tests.py b/tests/regressiontests/cache/tests.py index 264ef74abd..b85c8d4cf3 100644 --- a/tests/regressiontests/cache/tests.py +++ b/tests/regressiontests/cache/tests.py @@ -266,6 +266,10 @@ class BaseCacheTests(object): self.assertEqual(self.cache.get('answer'), 32) self.assertRaises(ValueError, self.cache.decr, 'does_not_exist') + def test_close(self): + self.assertTrue(hasattr(self.cache, 'close')) + self.cache.close() + def test_data_types(self): # Many different data types can be cached stuff = { From 502be865c68635d5c31fa3fa58162b48412153ad Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Oct 2012 10:31:14 +0100 Subject: [PATCH 002/302] Add 'page_kwarg' attribute to `MultipleObjectMixin`, removing hardcoded 'page'. --- django/views/generic/list.py | 4 +++- docs/ref/class-based-views/mixins-multiple-object.txt | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/django/views/generic/list.py b/django/views/generic/list.py index ec30c58f29..0c383d609a 100644 --- a/django/views/generic/list.py +++ b/django/views/generic/list.py @@ -17,6 +17,7 @@ class MultipleObjectMixin(ContextMixin): paginate_by = None context_object_name = None paginator_class = Paginator + page_kwarg = 'page' def get_queryset(self): """ @@ -39,7 +40,8 @@ class MultipleObjectMixin(ContextMixin): Paginate the queryset, if needed. """ paginator = self.get_paginator(queryset, page_size, allow_empty_first_page=self.get_allow_empty()) - page = self.kwargs.get('page') or self.request.GET.get('page') or 1 + page_kwarg = self.page_kwarg + page = self.kwargs.get(page_kwarg) or self.request.GET.get(page_kwarg) or 1 try: page_number = int(page) except ValueError: diff --git a/docs/ref/class-based-views/mixins-multiple-object.txt b/docs/ref/class-based-views/mixins-multiple-object.txt index cdb743fcbd..e6abf26e6a 100644 --- a/docs/ref/class-based-views/mixins-multiple-object.txt +++ b/docs/ref/class-based-views/mixins-multiple-object.txt @@ -69,8 +69,15 @@ MultipleObjectMixin An integer specifying how many objects should be displayed per page. If this is given, the view will paginate objects with :attr:`MultipleObjectMixin.paginate_by` objects per page. The view will - expect either a ``page`` query string parameter (via ``GET``) or a - ``page`` variable specified in the URLconf. + expect either a ``page`` query string parameter (via ``request.GET``) + or a ``page`` variable specified in the URLconf. + + .. attribute:: page_kwarg + + A string specifying the name to use for the page parameter. + The view will expect this prameter to be available either as a query + string parameter (via ``request.GET``) or as a kwarg variable specified + in the URLconf. Defaults to ``"page"``. .. attribute:: paginator_class From f824a951776db86c04aaefc1e7c1c12ffb84c798 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 25 Oct 2012 13:19:53 +0100 Subject: [PATCH 003/302] Test for `ListView.page_kwarg` --- tests/regressiontests/generic_views/list.py | 10 ++++++++++ tests/regressiontests/generic_views/urls.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/tests/regressiontests/generic_views/list.py b/tests/regressiontests/generic_views/list.py index 14dc1d725d..d267824b84 100644 --- a/tests/regressiontests/generic_views/list.py +++ b/tests/regressiontests/generic_views/list.py @@ -99,6 +99,16 @@ class ListViewTests(TestCase): # Custom pagination allows for 2 orphans on a page size of 5 self.assertEqual(len(res.context['object_list']), 7) + def test_paginated_custom_page_kwarg(self): + self._make_authors(100) + res = self.client.get('/list/authors/paginated/custom_page_kwarg/', {'pagina': '2'}) + self.assertEqual(res.status_code, 200) + self.assertTemplateUsed(res, 'generic_views/author_list.html') + self.assertEqual(len(res.context['object_list']), 30) + self.assertIs(res.context['author_list'], res.context['object_list']) + self.assertEqual(res.context['author_list'][0].name, 'Author 30') + self.assertEqual(res.context['page_obj'].number, 2) + def test_paginated_custom_paginator_constructor(self): self._make_authors(7) res = self.client.get('/list/authors/paginated/custom_constructor/') diff --git a/tests/regressiontests/generic_views/urls.py b/tests/regressiontests/generic_views/urls.py index c72bfecb65..a212b830a5 100644 --- a/tests/regressiontests/generic_views/urls.py +++ b/tests/regressiontests/generic_views/urls.py @@ -149,6 +149,8 @@ urlpatterns = patterns('', views.AuthorList.as_view(queryset=None)), (r'^list/authors/paginated/custom_class/$', views.AuthorList.as_view(paginate_by=5, paginator_class=views.CustomPaginator)), + (r'^list/authors/paginated/custom_page_kwarg/$', + views.AuthorList.as_view(paginate_by=30, page_kwarg='pagina')), (r'^list/authors/paginated/custom_constructor/$', views.AuthorListCustomPaginator.as_view()), From 90c76564669fa03caefcf4318ffdf9ba8fa4d40b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 26 Oct 2012 10:23:33 +0200 Subject: [PATCH 004/302] Fixed #19191 -- Corrected a typo in CustomUser docs Thanks spleeyah for the report. --- docs/topics/auth.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/auth.txt b/docs/topics/auth.txt index 41159984f6..d261a3c90b 100644 --- a/docs/topics/auth.txt +++ b/docs/topics/auth.txt @@ -1889,7 +1889,7 @@ password resets. You must then provide some key implementation details: as the identifying field:: class MyUser(AbstractBaseUser): - identfier = models.CharField(max_length=40, unique=True, db_index=True) + identifier = models.CharField(max_length=40, unique=True, db_index=True) ... USERNAME_FIELD = 'identifier' From be29329ccd49a84d3f7238111aebf97c4aaac581 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 26 Oct 2012 20:44:00 +0200 Subject: [PATCH 005/302] Fixed #16820 -- Treated '0' value as True for checkbox inputs Thanks Dan Fairs for the report and the initial patch. --- django/forms/widgets.py | 2 +- tests/regressiontests/forms/tests/forms.py | 5 +++++ tests/regressiontests/forms/tests/widgets.py | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/django/forms/widgets.py b/django/forms/widgets.py index 763da0cff2..061988c6c0 100644 --- a/django/forms/widgets.py +++ b/django/forms/widgets.py @@ -528,7 +528,7 @@ class CheckboxInput(Widget): values = {'true': True, 'false': False} if isinstance(value, six.string_types): value = values.get(value.lower(), value) - return value + return bool(value) def _has_changed(self, initial, data): # Sometimes data or initial could be None or '' which should be the diff --git a/tests/regressiontests/forms/tests/forms.py b/tests/regressiontests/forms/tests/forms.py index a8a28ba806..1c83ed04d4 100644 --- a/tests/regressiontests/forms/tests/forms.py +++ b/tests/regressiontests/forms/tests/forms.py @@ -269,6 +269,11 @@ class FormsTestCase(TestCase): f = SignupForm({'email': 'test@example.com', 'get_spam': 'false'}, auto_id=False) self.assertHTMLEqual(str(f['get_spam']), '') + # A value of '0' should be interpreted as a True value (#16820) + f = SignupForm({'email': 'test@example.com', 'get_spam': '0'}) + self.assertTrue(f.is_valid()) + self.assertTrue(f.cleaned_data.get('get_spam')) + def test_widget_output(self): # Any Field can have a Widget class passed to its constructor: class ContactForm(Form): diff --git a/tests/regressiontests/forms/tests/widgets.py b/tests/regressiontests/forms/tests/widgets.py index 4bdd3f76ea..88469a79e8 100644 --- a/tests/regressiontests/forms/tests/widgets.py +++ b/tests/regressiontests/forms/tests/widgets.py @@ -225,6 +225,10 @@ class FormsWidgetTestCase(TestCase): # checkboxes). self.assertFalse(w.value_from_datadict({}, {}, 'testing')) + value = w.value_from_datadict({'testing': '0'}, {}, 'testing') + self.assertIsInstance(value, bool) + self.assertTrue(value) + self.assertFalse(w._has_changed(None, None)) self.assertFalse(w._has_changed(None, '')) self.assertFalse(w._has_changed('', None)) From 195bc37f1d251eb7b5bc4d5f916f3b08745a504f Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 26 Oct 2012 22:09:48 +0200 Subject: [PATCH 006/302] Fixed httpwrappers tests under hash randomization --- tests/regressiontests/httpwrappers/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/regressiontests/httpwrappers/tests.py b/tests/regressiontests/httpwrappers/tests.py index 553b6ecff4..2d172ad0e0 100644 --- a/tests/regressiontests/httpwrappers/tests.py +++ b/tests/regressiontests/httpwrappers/tests.py @@ -149,7 +149,7 @@ class QueryDictTests(unittest.TestCase): self.assertEqual(q.setdefault('foo', 'bar'), 'bar') self.assertEqual(q['foo'], 'bar') self.assertEqual(q.getlist('foo'), ['bar']) - self.assertEqual(q.urlencode(), 'foo=bar&name=john') + self.assertIn(q.urlencode(), ['foo=bar&name=john', 'name=john&foo=bar']) q.clear() self.assertEqual(len(q), 0) @@ -266,14 +266,18 @@ class HttpResponseTests(unittest.TestCase): # The response also converts unicode or bytes keys to strings, but requires # them to contain ASCII r = HttpResponse() + del r['Content-Type'] r['foo'] = 'bar' l = list(r.items()) + self.assertEqual(len(l), 1) self.assertEqual(l[0], ('foo', 'bar')) self.assertIsInstance(l[0][0], str) r = HttpResponse() + del r['Content-Type'] r[b'foo'] = 'bar' l = list(r.items()) + self.assertEqual(len(l), 1) self.assertEqual(l[0], ('foo', 'bar')) self.assertIsInstance(l[0][0], str) From 46d27a6b8d8e988f665d102a1504e178133f7668 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 26 Oct 2012 22:40:35 +0200 Subject: [PATCH 007/302] Fixed feedgenerator tests under hash randomization --- tests/regressiontests/utils/feedgenerator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/regressiontests/utils/feedgenerator.py b/tests/regressiontests/utils/feedgenerator.py index b646a5997d..bcd53bb2a0 100644 --- a/tests/regressiontests/utils/feedgenerator.py +++ b/tests/regressiontests/utils/feedgenerator.py @@ -104,10 +104,14 @@ class FeedgeneratorTest(unittest.TestCase): feed = feedgenerator.Rss201rev2Feed('title', '/link/', 'descr') self.assertEqual(feed.feed['feed_url'], None) feed_content = feed.writeString('utf-8') - self.assertNotIn('', feed_content) + self.assertIn(' Date: Fri, 26 Oct 2012 22:47:45 +0200 Subject: [PATCH 008/302] Fixed comment_test tests under hash randomization. Thanks clelland for the patch. --- .../comment_tests/tests/feed_tests.py | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/regressiontests/comment_tests/tests/feed_tests.py b/tests/regressiontests/comment_tests/tests/feed_tests.py index 1ec316eab8..c93d7abf7d 100644 --- a/tests/regressiontests/comment_tests/tests/feed_tests.py +++ b/tests/regressiontests/comment_tests/tests/feed_tests.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +from xml.etree import ElementTree as ET + from django.conf import settings from django.contrib.comments.models import Comment from django.contrib.contenttypes.models import ContentType @@ -31,8 +33,21 @@ class CommentFeedTests(CommentTestCase): response = self.client.get(self.feed_url) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Type'], 'application/rss+xml; charset=utf-8') - self.assertContains(response, '') - self.assertContains(response, 'example.com comments') - self.assertContains(response, 'http://example.com/') - self.assertContains(response, '') + + rss_elem = ET.fromstring(response.content) + + self.assertEqual(rss_elem.tag, "rss") + self.assertEqual(rss_elem.attrib, {"version": "2.0"}) + + channel_elem = rss_elem.find("channel") + + title_elem = channel_elem.find("title") + self.assertEqual(title_elem.text, "example.com comments") + + link_elem = channel_elem.find("link") + self.assertEqual(link_elem.text, "http://example.com/") + + atomlink_elem = channel_elem.find("{http://www.w3.org/2005/Atom}link") + self.assertEqual(atomlink_elem.attrib, {"href": "http://example.com/rss/comments/", "rel": "self"}) + self.assertNotContains(response, "A comment for the second site.") From 11699ac4b5f98ec11dba02b356a8fd4ab6b4b889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 25 Oct 2012 20:57:32 +0300 Subject: [PATCH 009/302] Fixed #19190 -- Refactored Query select clause attributes The Query.select and Query.select_fields were collapsed into one list because the attributes had to be always in sync. Now that they are in one attribute it is impossible to edit them out of sync. Similar collapse was done for Query.related_select_cols and Query.related_select_fields. --- django/contrib/gis/db/models/sql/compiler.py | 4 +- django/db/models/sql/compiler.py | 28 +++---- django/db/models/sql/constants.py | 3 + django/db/models/sql/query.py | 87 +++++++++----------- django/db/models/sql/subqueries.py | 4 +- 5 files changed, 61 insertions(+), 65 deletions(-) diff --git a/django/contrib/gis/db/models/sql/compiler.py b/django/contrib/gis/db/models/sql/compiler.py index cf6a8ad047..0dcf50d32a 100644 --- a/django/contrib/gis/db/models/sql/compiler.py +++ b/django/contrib/gis/db/models/sql/compiler.py @@ -39,7 +39,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): if self.query.select: only_load = self.deferred_to_columns() # This loop customized for GeoQuery. - for col, field in zip(self.query.select, self.query.select_fields): + for col, field in self.query.select: if isinstance(col, (list, tuple)): alias, column = col table = self.query.alias_map[alias].table_name @@ -85,7 +85,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): ]) # This loop customized for GeoQuery. - for (table, col), field in zip(self.query.related_select_cols, self.query.related_select_fields): + for (table, col), field in self.query.related_select_cols: r = self.get_field_select(field, table, col) if with_aliases and col in col_aliases: c_alias = 'Col%d' % len(col_aliases) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index b9095e503a..7461f5f31d 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -6,7 +6,7 @@ from django.db.backends.util import truncate_name from django.db.models.constants import LOOKUP_SEP from django.db.models.query_utils import select_related_descend from django.db.models.sql.constants import (SINGLE, MULTI, ORDER_DIR, - GET_ITERATOR_CHUNK_SIZE) + GET_ITERATOR_CHUNK_SIZE, SelectInfo) from django.db.models.sql.datastructures import EmptyResultSet from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.query import get_order_dir, Query @@ -188,7 +188,7 @@ class SQLCompiler(object): col_aliases = set() if self.query.select: only_load = self.deferred_to_columns() - for col in self.query.select: + for col, _ in self.query.select: if isinstance(col, (list, tuple)): alias, column = col table = self.query.alias_map[alias].table_name @@ -233,7 +233,7 @@ class SQLCompiler(object): for alias, aggregate in self.query.aggregate_select.items() ]) - for table, col in self.query.related_select_cols: + for (table, col), _ in self.query.related_select_cols: r = '%s.%s' % (qn(table), qn(col)) if with_aliases and col in col_aliases: c_alias = 'Col%d' % len(col_aliases) @@ -557,8 +557,9 @@ class SQLCompiler(object): for extra_select, extra_params in six.itervalues(self.query.extra_select): extra_selects.append(extra_select) params.extend(extra_params) - cols = (group_by + self.query.select + - self.query.related_select_cols + extra_selects) + select_cols = [s.col for s in self.query.select] + related_select_cols = [s.col for s in self.query.related_select_cols] + cols = (group_by + select_cols + related_select_cols + extra_selects) seen = set() for col in cols: if col in seen: @@ -589,7 +590,6 @@ class SQLCompiler(object): opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] - self.query.related_select_fields = [] if not used: used = set() if dupe_set is None: @@ -664,8 +664,8 @@ class SQLCompiler(object): used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) - self.query.related_select_cols.extend(columns) - self.query.related_select_fields.extend(f.rel.to._meta.fields) + self.query.related_select_cols.extend( + SelectInfo(col, field) for col, field in zip(columns, f.rel.to._meta.fields)) if restricted: next = requested.get(f.name, {}) else: @@ -734,8 +734,8 @@ class SQLCompiler(object): used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, local_only=True) - self.query.related_select_cols.extend(columns) - self.query.related_select_fields.extend(model._meta.fields) + self.query.related_select_cols.extend( + SelectInfo(col, field) for col, field in zip(columns, model._meta.fields)) next = requested.get(f.related_query_name(), {}) # Use True here because we are looking at the _reverse_ side of @@ -772,7 +772,7 @@ class SQLCompiler(object): if resolve_columns: if fields is None: # We only set this up here because - # related_select_fields isn't populated until + # related_select_cols isn't populated until # execute_sql() has been called. # We also include types of fields of related models that @@ -782,11 +782,11 @@ class SQLCompiler(object): # This code duplicates the logic for the order of fields # found in get_columns(). It would be nice to clean this up. - if self.query.select_fields: - fields = self.query.select_fields + if self.query.select: + fields = [f.field for f in self.query.select] else: fields = self.query.model._meta.fields - fields = fields + self.query.related_select_fields + fields = fields + [f.field for f in self.query.related_select_cols] # If the field was deferred, exclude it from being passed # into `resolve_columns` because it wasn't selected. diff --git a/django/db/models/sql/constants.py b/django/db/models/sql/constants.py index f750310624..7e34047e1d 100644 --- a/django/db/models/sql/constants.py +++ b/django/db/models/sql/constants.py @@ -25,6 +25,9 @@ JoinInfo = namedtuple('JoinInfo', 'table_name rhs_alias join_type lhs_alias ' 'lhs_join_col rhs_join_col nullable') +# Pairs of column clauses to select, and (possibly None) field for the clause. +SelectInfo = namedtuple('SelectInfo', 'col field') + # How many results to expect from a cursor.execute call MULTI = 'multi' SINGLE = 'single' diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index cef01c48ab..de7e5904a3 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -20,7 +20,7 @@ from django.db.models.expressions import ExpressionNode from django.db.models.fields import FieldDoesNotExist from django.db.models.sql import aggregates as base_aggregates_module from django.db.models.sql.constants import (QUERY_TERMS, ORDER_DIR, SINGLE, - ORDER_PATTERN, JoinInfo) + ORDER_PATTERN, JoinInfo, SelectInfo) from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode, @@ -115,17 +115,20 @@ class Query(object): self.default_ordering = True self.standard_ordering = True self.ordering_aliases = [] - self.related_select_fields = [] self.dupe_avoidance = {} self.used_aliases = set() self.filter_is_sticky = False self.included_inherited_models = {} - # SQL-related attributes + # SQL-related attributes + # Select and related select clauses as SelectInfo instances. + # The select is used for cases where we want to set up the select + # clause to contain other than default fields (values(), annotate(), + # subqueries...) self.select = [] - # For each to-be-selected field in self.select there must be a - # corresponding entry in self.select - git seems to need this. - self.select_fields = [] + # The related_select_cols is used for columns needed for + # select_related - this is populated in compile stage. + self.related_select_cols = [] self.tables = [] # Aliases in the order they are created. self.where = where() self.where_class = where @@ -138,7 +141,6 @@ class Query(object): self.select_for_update = False self.select_for_update_nowait = False self.select_related = False - self.related_select_cols = [] # SQL aggregate-related attributes self.aggregates = SortedDict() # Maps alias -> SQL aggregate function @@ -191,15 +193,14 @@ class Query(object): Pickling support. """ obj_dict = self.__dict__.copy() - obj_dict['related_select_fields'] = [] obj_dict['related_select_cols'] = [] # Fields can't be pickled, so if a field list has been # specified, we pickle the list of field names instead. # None is also a possible value; that can pass as-is - obj_dict['select_fields'] = [ - f is not None and f.name or None - for f in obj_dict['select_fields'] + obj_dict['select'] = [ + (s.col, s.field is not None and s.field.name or None) + for s in obj_dict['select'] ] return obj_dict @@ -209,9 +210,9 @@ class Query(object): """ # Rebuild list of field instances opts = obj_dict['model']._meta - obj_dict['select_fields'] = [ - name is not None and opts.get_field(name) or None - for name in obj_dict['select_fields'] + obj_dict['select'] = [ + SelectInfo(tpl[0], tpl[1] is not None and opts.get_field(tpl[1]) or None) + for tpl in obj_dict['select'] ] self.__dict__.update(obj_dict) @@ -256,10 +257,9 @@ class Query(object): obj.standard_ordering = self.standard_ordering obj.included_inherited_models = self.included_inherited_models.copy() obj.ordering_aliases = [] - obj.select_fields = self.select_fields[:] - obj.related_select_fields = self.related_select_fields[:] obj.dupe_avoidance = self.dupe_avoidance.copy() obj.select = self.select[:] + obj.related_select_cols = [] obj.tables = self.tables[:] obj.where = copy.deepcopy(self.where, memo=memo) obj.where_class = self.where_class @@ -275,7 +275,6 @@ class Query(object): obj.select_for_update = self.select_for_update obj.select_for_update_nowait = self.select_for_update_nowait obj.select_related = self.select_related - obj.related_select_cols = [] obj.aggregates = copy.deepcopy(self.aggregates, memo=memo) if self.aggregate_select_mask is None: obj.aggregate_select_mask = None @@ -384,7 +383,6 @@ class Query(object): query.select_for_update = False query.select_related = False query.related_select_cols = [] - query.related_select_fields = [] result = query.get_compiler(using).execute_sql(SINGLE) if result is None: @@ -527,14 +525,14 @@ class Query(object): # Selection columns and extra extensions are those provided by 'rhs'. self.select = [] - for col in rhs.select: + for col, field in rhs.select: if isinstance(col, (list, tuple)): - self.select.append((change_map.get(col[0], col[0]), col[1])) + new_col = change_map.get(col[0], col[0]), col[1] + self.select.append(SelectInfo(new_col, field)) else: item = copy.deepcopy(col) item.relabel_aliases(change_map) - self.select.append(item) - self.select_fields = rhs.select_fields[:] + self.select.append(SelectInfo(item, field)) if connector == OR: # It would be nice to be able to handle this, but the queries don't @@ -750,24 +748,23 @@ class Query(object): """ assert set(change_map.keys()).intersection(set(change_map.values())) == set() + def relabel_column(col): + if isinstance(col, (list, tuple)): + old_alias = col[0] + return (change_map.get(old_alias, old_alias), col[1]) + else: + col.relabel_aliases(change_map) + return col # 1. Update references in "select" (normal columns plus aliases), # "group by", "where" and "having". self.where.relabel_aliases(change_map) self.having.relabel_aliases(change_map) - for columns in [self.select, self.group_by or []]: - for pos, col in enumerate(columns): - if isinstance(col, (list, tuple)): - old_alias = col[0] - columns[pos] = (change_map.get(old_alias, old_alias), col[1]) - else: - col.relabel_aliases(change_map) - for mapping in [self.aggregates]: - for key, col in mapping.items(): - if isinstance(col, (list, tuple)): - old_alias = col[0] - mapping[key] = (change_map.get(old_alias, old_alias), col[1]) - else: - col.relabel_aliases(change_map) + if self.group_by: + self.group_by = [relabel_column(col) for col in self.group_by] + self.select = [SelectInfo(relabel_column(s.col), s.field) + for s in self.select] + self.aggregates = SortedDict( + (key, relabel_column(col)) for key, col in self.aggregates.items()) # 2. Rename the alias in the internal table/alias datastructures. for k, aliases in self.join_map.items(): @@ -1570,7 +1567,7 @@ class Query(object): # since we are adding a IN clause. This prevents the # database from tripping over IN (...,NULL,...) selects and returning # nothing - alias, col = query.select[0] + alias, col = query.select[0].col query.where.add((Constraint(alias, col, None), 'isnull', False), AND) self.add_filter(('%s__in' % prefix, query), negate=True, trim=True, @@ -1629,7 +1626,6 @@ class Query(object): Removes all fields from SELECT clause. """ self.select = [] - self.select_fields = [] self.default_cols = False self.select_related = False self.set_extra_mask(()) @@ -1642,7 +1638,6 @@ class Query(object): columns. """ self.select = [] - self.select_fields = [] def add_distinct_fields(self, *field_names): """ @@ -1674,8 +1669,7 @@ class Query(object): col = join.lhs_join_col joins = joins[:-1] self.promote_joins(joins[1:]) - self.select.append((final_alias, col)) - self.select_fields.append(field) + self.select.append(SelectInfo((final_alias, col), field)) except MultiJoin: raise FieldError("Invalid field name: '%s'" % name) except FieldError: @@ -1731,8 +1725,8 @@ class Query(object): """ self.group_by = [] - for sel in self.select: - self.group_by.append(sel) + for col, _ in self.select: + self.group_by.append(col) def add_count_column(self): """ @@ -1745,7 +1739,7 @@ class Query(object): else: assert len(self.select) == 1, \ "Cannot add count col with multiple cols in 'select': %r" % self.select - count = self.aggregates_module.Count(self.select[0]) + count = self.aggregates_module.Count(self.select[0].col) else: opts = self.model._meta if not self.select: @@ -1757,7 +1751,7 @@ class Query(object): assert len(self.select) == 1, \ "Cannot add count col with multiple cols in 'select'." - count = self.aggregates_module.Count(self.select[0], distinct=True) + count = self.aggregates_module.Count(self.select[0].col, distinct=True) # Distinct handling is done in Count(), so don't do it at this # level. self.distinct = False @@ -1781,7 +1775,6 @@ class Query(object): d = d.setdefault(part, {}) self.select_related = field_dict self.related_select_cols = [] - self.related_select_fields = [] def add_extra(self, select, select_params, where, params, tables, order_by): """ @@ -1975,7 +1968,7 @@ class Query(object): self.unref_alias(select_alias) select_alias = join_info.rhs_alias select_col = join_info.rhs_join_col - self.select = [(select_alias, select_col)] + self.select = [SelectInfo((select_alias, select_col), None)] self.remove_inherited_models() def is_nullable(self, field): diff --git a/django/db/models/sql/subqueries.py b/django/db/models/sql/subqueries.py index 24ac957cbf..39d1ee0116 100644 --- a/django/db/models/sql/subqueries.py +++ b/django/db/models/sql/subqueries.py @@ -76,7 +76,7 @@ class DeleteQuery(Query): return else: innerq.clear_select_clause() - innerq.select, innerq.select_fields = [(self.get_initial_alias(), pk.column)], [None] + innerq.select = [SelectInfo((self.get_initial_alias(), pk.column), None)] values = innerq where = self.where_class() where.add((Constraint(None, pk.column, pk), 'in', values), AND) @@ -244,7 +244,7 @@ class DateQuery(Query): alias = result[3][-1] select = Date((alias, field.column), lookup_type) self.clear_select_clause() - self.select, self.select_fields = [select], [None] + self.select = [SelectInfo(select, None)] self.distinct = True self.order_by = order == 'ASC' and [1] or [-1] From 373df56d36891b9ab1f88519bf9e8f3c0b3bb108 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Fri, 26 Oct 2012 22:01:34 -0300 Subject: [PATCH 010/302] Advanced version identifiers for 1.6 cycle. --- django/__init__.py | 2 +- docs/conf.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/django/__init__.py b/django/__init__.py index 32e1374765..873c328add 100644 --- a/django/__init__.py +++ b/django/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 5, 0, 'alpha', 0) +VERSION = (1, 6, 0, 'alpha', 0) def get_version(*args, **kwargs): # Don't litter django/__init__.py with all the get_version stuff. diff --git a/docs/conf.py b/docs/conf.py index 433fd679a1..6dd84cffba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,11 +52,23 @@ copyright = 'Django Software Foundation and contributors' # built documents. # # The short X.Y version. -version = '1.5' +version = '1.6' # The full version, including alpha/beta/rc tags. -release = '1.5' +try: + from django import VERSION, get_version +except ImportError: + release = version +else: + def django_release(): + pep386ver = get_version() + if VERSION[3:5] == ('alpha', 0) and 'dev' not in pep386ver: + return pep386ver + '.dev' + return pep386ver + + release = django_release() + # The next version to be released -django_next_version = '1.6' +django_next_version = '1.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 2249bd275cbae6b73716bf36f5f36def1bb4222e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sat, 27 Oct 2012 04:51:14 +0300 Subject: [PATCH 011/302] Fixed Oracle failure for "%" in table name --- django/db/backends/oracle/base.py | 4 ++++ tests/regressiontests/backends/tests.py | 8 ++++++++ tests/regressiontests/inspectdb/tests.py | 11 +++++++---- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index 6bf2e815a7..aad52992dd 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -256,6 +256,10 @@ WHEN (new.%(col_name)s IS NULL) if not name.startswith('"') and not name.endswith('"'): name = '"%s"' % util.truncate_name(name.upper(), self.max_name_length()) + # Oracle puts the query text into a (query % args) construct, so % signs + # in names need to be escaped. The '%%' will be collapsed back to '%' at + # that stage so we aren't really making the name longer here. + name = name.replace('%','%%') return name.upper() def random_function_sql(self): diff --git a/tests/regressiontests/backends/tests.py b/tests/regressiontests/backends/tests.py index d284cfaac8..4766dcf44e 100644 --- a/tests/regressiontests/backends/tests.py +++ b/tests/regressiontests/backends/tests.py @@ -25,6 +25,14 @@ from . import models class OracleChecks(unittest.TestCase): + @unittest.skipUnless(connection.vendor == 'oracle', + "No need to check Oracle quote_name semantics") + def test_quote_name(self): + # Check that '%' chars are escaped for query execution. + name = '"SOME%NAME"' + quoted_name = connection.ops.quote_name(name) + self.assertEquals(quoted_name % (), name) + @unittest.skipUnless(connection.vendor == 'oracle', "No need to check Oracle cursor semantics") def test_dbms_session(self): diff --git a/tests/regressiontests/inspectdb/tests.py b/tests/regressiontests/inspectdb/tests.py index 484e7f4060..028d263337 100644 --- a/tests/regressiontests/inspectdb/tests.py +++ b/tests/regressiontests/inspectdb/tests.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.core.management import call_command +from django.db import connection from django.test import TestCase, skipUnlessDBFeature from django.utils.six import StringIO @@ -60,14 +61,16 @@ class InspectDBTestCase(TestCase): self.assertIn("number_45extra = models.CharField", output) def test_special_column_name_introspection(self): - """Introspection of column names containing special characters, - unsuitable for Python identifiers + """ + Introspection of column names containing special characters, + unsuitable for Python identifiers """ out = StringIO() call_command('inspectdb', stdout=out) output = out.getvalue() + base_name = 'Field' if connection.vendor != 'oracle' else 'field' self.assertIn("field = models.IntegerField()", output) - self.assertIn("field_field = models.IntegerField(db_column='Field_')", output) - self.assertIn("field_field_0 = models.IntegerField(db_column='Field__')", output) + self.assertIn("field_field = models.IntegerField(db_column='%s_')" % base_name, output) + self.assertIn("field_field_0 = models.IntegerField(db_column='%s__')" % base_name, output) self.assertIn("field_field_1 = models.IntegerField(db_column='__field')", output) self.assertIn("prc_x = models.IntegerField(db_column='prc(%) x')", output) From c159d9cec0baab7bbd04d5d51a92a51e354a722a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sat, 27 Oct 2012 04:52:12 +0300 Subject: [PATCH 012/302] Fixed Oracle failure caused by None converted to '' in select_related case --- django/db/models/query.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index dc1ddf1606..da4c69f362 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -1418,8 +1418,15 @@ def get_cached_row(row, index_start, using, klass_info, offset=0): fields = row[index_start : index_start + field_count] # If all the select_related columns are None, then the related # object must be non-existent - set the relation to None. - # Otherwise, construct the related object. - if fields == (None,) * field_count: + # Otherwise, construct the related object. Also, some backends treat '' + # and None equivalently for char fields, so we have to be prepared for + # '' values. + if connections[using].features.interprets_empty_strings_as_nulls: + vals = tuple([None if f == '' else f for f in fields]) + else: + vals = fields + + if vals == (None,) * field_count: obj = None else: if field_names: From a5152bb64677fb9b976f6b35e80b11b368ef1e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sat, 27 Oct 2012 18:10:02 +0300 Subject: [PATCH 013/302] Marked a test as expectedFailure on Oracle --- tests/regressiontests/introspection/tests.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/regressiontests/introspection/tests.py b/tests/regressiontests/introspection/tests.py index a54e0c670b..4b8a3277e2 100644 --- a/tests/regressiontests/introspection/tests.py +++ b/tests/regressiontests/introspection/tests.py @@ -4,10 +4,15 @@ from functools import update_wrapper from django.db import connection from django.test import TestCase, skipUnlessDBFeature, skipIfDBFeature -from django.utils import six +from django.utils import six, unittest from .models import Reporter, Article +if connection.vendor == 'oracle': + expectedFailureOnOracle = unittest.expectedFailure +else: + expectedFailureOnOracle = lambda f: f + # # The introspection module is optional, so methods tested here might raise # NotImplementedError. This is perfectly acceptable behavior for the backend @@ -89,7 +94,13 @@ class IntrospectionTests(six.with_metaclass(IgnoreNotimplementedError, TestCase) [datatype(r[1], r) for r in desc], ['IntegerField', 'CharField', 'CharField', 'CharField', 'BigIntegerField'] ) - # Check also length of CharFields + + # The following test fails on Oracle due to #17202 (can't correctly + # inspect the length of character columns). + @expectedFailureOnOracle + def test_get_table_description_col_lengths(self): + cursor = connection.cursor() + desc = connection.introspection.get_table_description(cursor, Reporter._meta.db_table) self.assertEqual( [r[3] for r in desc if datatype(r[1], r) == 'CharField'], [30, 30, 75] From b55de81b9e4d820a223c066022050a0df0ee1dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sat, 27 Oct 2012 18:10:41 +0300 Subject: [PATCH 014/302] Ensured gis tests aren't run on non-gis Oracle --- django/contrib/gis/tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/gis/tests/utils.py b/django/contrib/gis/tests/utils.py index a83ba8a93f..8355b27fd7 100644 --- a/django/contrib/gis/tests/utils.py +++ b/django/contrib/gis/tests/utils.py @@ -26,7 +26,7 @@ mysql = _default_db == 'mysql' spatialite = _default_db == 'spatialite' HAS_SPATIALREFSYS = True -if oracle: +if oracle and 'gis' in settings.DATABASES[DEFAULT_DB_ALIAS]['ENGINE']: from django.contrib.gis.db.backends.oracle.models import SpatialRefSys elif postgis: from django.contrib.gis.db.backends.postgis.models import SpatialRefSys From 83ba0a9d4b078fd177ac5c06699486d708d62bff Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 27 Oct 2012 11:49:46 +0200 Subject: [PATCH 015/302] Fixed #18978 -- Moved cleanup command to sessions. This removes a dependency of 'core' on 'contrib'. --- django/bin/daily_cleanup.py | 8 +++++++- django/contrib/sessions/management/__init__.py | 0 .../sessions/management/commands/__init__.py | 0 .../management/commands/clearsessions.py | 11 +++++++++++ django/core/management/commands/cleanup.py | 16 ++++++++-------- docs/internals/deprecation.txt | 5 +++++ docs/ref/django-admin.txt | 15 +++++++++++++++ docs/releases/1.5.txt | 13 ++++++++++++- docs/topics/http/sessions.txt | 2 +- 9 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 django/contrib/sessions/management/__init__.py create mode 100644 django/contrib/sessions/management/commands/__init__.py create mode 100644 django/contrib/sessions/management/commands/clearsessions.py diff --git a/django/bin/daily_cleanup.py b/django/bin/daily_cleanup.py index c9f4cb905c..ac3de00f2c 100755 --- a/django/bin/daily_cleanup.py +++ b/django/bin/daily_cleanup.py @@ -7,7 +7,13 @@ Can be run as a cronjob to clean out old data from the database (only expired sessions at the moment). """ +import warnings + from django.core import management if __name__ == "__main__": - management.call_command('cleanup') + warnings.warn( + "The `daily_cleanup` script has been deprecated " + "in favor of `django-admin.py clearsessions`.", + PendingDeprecationWarning) + management.call_command('clearsessions') diff --git a/django/contrib/sessions/management/__init__.py b/django/contrib/sessions/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django/contrib/sessions/management/commands/__init__.py b/django/contrib/sessions/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/django/contrib/sessions/management/commands/clearsessions.py b/django/contrib/sessions/management/commands/clearsessions.py new file mode 100644 index 0000000000..fb7666f85a --- /dev/null +++ b/django/contrib/sessions/management/commands/clearsessions.py @@ -0,0 +1,11 @@ +from django.core.management.base import NoArgsCommand +from django.utils import timezone + +class Command(NoArgsCommand): + help = "Can be run as a cronjob or directly to clean out expired sessions (only with the database backend at the moment)." + + def handle_noargs(self, **options): + from django.db import transaction + from django.contrib.sessions.models import Session + Session.objects.filter(expire_date__lt=timezone.now()).delete() + transaction.commit_unless_managed() diff --git a/django/core/management/commands/cleanup.py b/django/core/management/commands/cleanup.py index e19d1649be..f83c64be8f 100644 --- a/django/core/management/commands/cleanup.py +++ b/django/core/management/commands/cleanup.py @@ -1,11 +1,11 @@ -from django.core.management.base import NoArgsCommand -from django.utils import timezone +import warnings -class Command(NoArgsCommand): - help = "Can be run as a cronjob or directly to clean out old data from the database (only expired sessions at the moment)." +from django.contrib.sessions.management.commands import clearsessions + +class Command(clearsessions.Command): def handle_noargs(self, **options): - from django.db import transaction - from django.contrib.sessions.models import Session - Session.objects.filter(expire_date__lt=timezone.now()).delete() - transaction.commit_unless_managed() + warnings.warn( + "The `cleanup` command has been deprecated in favor of `clearsessions`.", + PendingDeprecationWarning) + super(Command, self).handle_noargs(**options) diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 10bbfe1a91..77371c8608 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -293,6 +293,11 @@ these changes. * The ``AUTH_PROFILE_MODULE`` setting, and the ``get_profile()`` method on the User model, will be removed. +* The ``cleanup`` management command will be removed. It's replaced by + ``clearsessions``. + +* The ``daily_cleanup.py`` script will be removed. + 2.0 --- diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 7fa7539985..833db0839c 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -96,6 +96,9 @@ cleanup Can be run as a cronjob or directly to clean out old data from the database (only expired sessions at the moment). +.. versionchanged:: 1.5 + :djadmin:`cleanup` is deprecated. Use :djadmin:`clearsessions` instead. + compilemessages --------------- @@ -1187,6 +1190,18 @@ This command is only available if :doc:`GeoDjango ` Please refer to its :djadmin:`description ` in the GeoDjango documentation. +``django.contrib.sessions`` +--------------------------- + +clearsessions +~~~~~~~~~~~~~~~ + +.. django-admin:: clearsessions + +Can be run as a cron job or directly to clean out expired sessions. + +This is only supported by the database backend at the moment. + ``django.contrib.sitemaps`` --------------------------- diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index a0ce3cc7a4..ebf88e83b9 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -613,7 +613,6 @@ Define a ``__str__`` method and apply the The :func:`~django.utils.itercompat.product` function has been deprecated. Use the built-in :func:`itertools.product` instead. - ``django.utils.markup`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -621,3 +620,15 @@ The markup contrib module has been deprecated and will follow an accelerated deprecation schedule. Direct use of python markup libraries or 3rd party tag libraries is preferred to Django maintaining this functionality in the framework. + +``cleanup`` management command +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :djadmin:`cleanup` management command has been deprecated and replaced by +:djadmin:`clearsessions`. + +``daily_cleanup.py`` script +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The undocumented ``daily_cleanup.py`` script has been deprecated. Use the +:djadmin:`clearsessions` management command instead. diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 15f9f7feba..0082b75db1 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -460,7 +460,7 @@ table. Django updates this row each time the session data changes. If the user logs out manually, Django deletes the row. But if the user does *not* log out, the row never gets deleted. -Django provides a sample clean-up script: ``django-admin.py cleanup``. +Django provides a sample clean-up script: ``django-admin.py clearsessions``. That script deletes any session in the session table whose ``expire_date`` is in the past -- but your application may have different requirements. From 04b00b668d0d56c37460cbed19671f4b1b5916c3 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 27 Oct 2012 19:18:03 +0200 Subject: [PATCH 016/302] Fixed #19200 -- Session expiry with cached_db Also did a little bit of cleanup. --- django/contrib/sessions/backends/base.py | 10 +++++-- django/contrib/sessions/backends/cached_db.py | 25 +++++++++++++--- django/contrib/sessions/backends/db.py | 2 +- .../sessions/backends/signed_cookies.py | 1 + django/contrib/sessions/tests.py | 29 ++++++++++++++----- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index c8393f23c6..65bbe059eb 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -170,9 +170,13 @@ class SessionBase(object): _session = property(_get_session) - def get_expiry_age(self): - """Get the number of seconds until the session expires.""" - expiry = self.get('_session_expiry') + def get_expiry_age(self, expiry=None): + """Get the number of seconds until the session expires. + + expiry is an optional parameter specifying the datetime of expiry. + """ + if expiry is None: + expiry = self.get('_session_expiry') if not expiry: # Checks both None and 0 cases return settings.SESSION_COOKIE_AGE if not isinstance(expiry, datetime): diff --git a/django/contrib/sessions/backends/cached_db.py b/django/contrib/sessions/backends/cached_db.py index ff6076df77..8e4cf8b7e7 100644 --- a/django/contrib/sessions/backends/cached_db.py +++ b/django/contrib/sessions/backends/cached_db.py @@ -2,9 +2,10 @@ Cached, database-backed sessions. """ -from django.conf import settings from django.contrib.sessions.backends.db import SessionStore as DBStore from django.core.cache import cache +from django.core.exceptions import SuspiciousOperation +from django.utils import timezone KEY_PREFIX = "django.contrib.sessions.cached_db" @@ -28,9 +29,21 @@ class SessionStore(DBStore): # Some backends (e.g. memcache) raise an exception on invalid # cache keys. If this happens, reset the session. See #17810. data = None + if data is None: - data = super(SessionStore, self).load() - cache.set(self.cache_key, data, settings.SESSION_COOKIE_AGE) + # Duplicate DBStore.load, because we need to keep track + # of the expiry date to set it properly in the cache. + try: + s = Session.objects.get( + session_key=self.session_key, + expire_date__gt=timezone.now() + ) + data = self.decode(s.session_data) + cache.set(self.cache_key, data, + self.get_expiry_age(s.expire_date)) + except (Session.DoesNotExist, SuspiciousOperation): + self.create() + data = {} return data def exists(self, session_key): @@ -40,7 +53,7 @@ class SessionStore(DBStore): def save(self, must_create=False): super(SessionStore, self).save(must_create) - cache.set(self.cache_key, self._session, settings.SESSION_COOKIE_AGE) + cache.set(self.cache_key, self._session, self.get_expiry_age()) def delete(self, session_key=None): super(SessionStore, self).delete(session_key) @@ -58,3 +71,7 @@ class SessionStore(DBStore): self.clear() self.delete(self.session_key) self.create() + + +# At bottom to avoid circular import +from django.contrib.sessions.models import Session diff --git a/django/contrib/sessions/backends/db.py b/django/contrib/sessions/backends/db.py index babdb72c27..4dacc96000 100644 --- a/django/contrib/sessions/backends/db.py +++ b/django/contrib/sessions/backends/db.py @@ -14,7 +14,7 @@ class SessionStore(SessionBase): def load(self): try: s = Session.objects.get( - session_key = self.session_key, + session_key=self.session_key, expire_date__gt=timezone.now() ) return self.decode(s.session_data) diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py index 41ba7af634..23915cf98c 100644 --- a/django/contrib/sessions/backends/signed_cookies.py +++ b/django/contrib/sessions/backends/signed_cookies.py @@ -32,6 +32,7 @@ class SessionStore(SessionBase): try: return signing.loads(self.session_key, serializer=PickleSerializer, + # This doesn't handle non-default expiry dates, see #19201 max_age=settings.SESSION_COOKIE_AGE, salt='django.contrib.sessions.backends.signed_cookies') except (signing.BadSignature, ValueError): diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index fc2d8753d7..527e8eb331 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -83,7 +83,7 @@ class SessionTestsMixin(object): self.session['some key'] = 1 self.session.modified = False self.session.accessed = False - self.assertTrue('some key' in self.session) + self.assertIn('some key', self.session) self.assertTrue(self.session.accessed) self.assertFalse(self.session.modified) @@ -200,28 +200,28 @@ class SessionTestsMixin(object): # Using seconds self.session.set_expiry(10) delta = self.session.get_expiry_date() - timezone.now() - self.assertTrue(delta.seconds in (9, 10)) + self.assertIn(delta.seconds, (9, 10)) age = self.session.get_expiry_age() - self.assertTrue(age in (9, 10)) + self.assertIn(age, (9, 10)) def test_custom_expiry_timedelta(self): # Using timedelta self.session.set_expiry(timedelta(seconds=10)) delta = self.session.get_expiry_date() - timezone.now() - self.assertTrue(delta.seconds in (9, 10)) + self.assertIn(delta.seconds, (9, 10)) age = self.session.get_expiry_age() - self.assertTrue(age in (9, 10)) + self.assertIn(age, (9, 10)) def test_custom_expiry_datetime(self): # Using fixed datetime self.session.set_expiry(timezone.now() + timedelta(seconds=10)) delta = self.session.get_expiry_date() - timezone.now() - self.assertTrue(delta.seconds in (9, 10)) + self.assertIn(delta.seconds, (9, 10)) age = self.session.get_expiry_age() - self.assertTrue(age in (9, 10)) + self.assertIn(age, (9, 10)) def test_custom_expiry_reset(self): self.session.set_expiry(None) @@ -258,6 +258,21 @@ class SessionTestsMixin(object): encoded = self.session.encode(data) self.assertEqual(self.session.decode(encoded), data) + def test_actual_expiry(self): + # Regression test for #19200 + old_session_key = None + new_session_key = None + try: + self.session['foo'] = 'bar' + self.session.set_expiry(-timedelta(seconds=10)) + self.session.create() + # With an expiry date in the past, the session expires instantly. + new_session = self.backend(self.session.session_key) + self.assertNotIn('foo', new_session) + finally: + self.session.delete(old_session_key) + self.session.delete(new_session_key) + class DatabaseSessionTests(SessionTestsMixin, TestCase): From b7d81715dc85fd5ec10a207b8b6ebca8399497e0 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Sat, 27 Oct 2012 21:18:48 +0200 Subject: [PATCH 017/302] Removed a redundant colon in the query docs. Thanks to Berker Peksag for the patch. --- docs/topics/db/queries.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/db/queries.txt b/docs/topics/db/queries.txt index d826a39562..321c8a42eb 100644 --- a/docs/topics/db/queries.txt +++ b/docs/topics/db/queries.txt @@ -417,7 +417,7 @@ translates (roughly) into the following SQL:: There's one exception though, in case of a :class:`~django.db.models.ForeignKey` you can specify the field name suffixed with ``_id``. In this case, the value parameter is expected - to contain the raw value of the foreign model's primary key. For example:: + to contain the raw value of the foreign model's primary key. For example: >>> Entry.objects.filter(blog_id__exact=4) From fc2681b22b120a468607c6aeb06163d8e5dbf897 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 27 Oct 2012 21:55:50 +0200 Subject: [PATCH 018/302] Fixed #17787 -- Documented reset caches by setting_changed signal --- docs/topics/testing.txt | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/topics/testing.txt b/docs/topics/testing.txt index d0b2e7cdf9..7c25a8b3ff 100644 --- a/docs/topics/testing.txt +++ b/docs/topics/testing.txt @@ -1586,15 +1586,23 @@ The decorator can also be applied to test case classes:: the original ``LoginTestCase`` is still equally affected by the decorator. -.. note:: +When overriding settings, make sure to handle the cases in which your app's +code uses a cache or similar feature that retains state even if the +setting is changed. Django provides the +:data:`django.test.signals.setting_changed` signal that lets you register +callbacks to clean up and otherwise reset state when settings are changed. - When overriding settings, make sure to handle the cases in which your app's - code uses a cache or similar feature that retains state even if the - setting is changed. Django provides the - :data:`django.test.signals.setting_changed` signal that lets you register - callbacks to clean up and otherwise reset state when settings are changed. - Note that this signal isn't currently used by Django itself, so changing - built-in settings may not yield the results you expect. +Django itself uses this signal to reset various data: + +=========================== ======================== +Overriden settings Data reset +=========================== ======================== +USE_TZ, TIME_ZONE Databases timezone +TEMPLATE_CONTEXT_PROCESSORS Context processors cache +TEMPLATE_LOADERS Template loaders cache +SERIALIZATION_MODULES Serializers cache +LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations +=========================== ======================== Emptying the test outbox ~~~~~~~~~~~~~~~~~~~~~~~~ From cd17a24083f3ef17cf4c40a41c9d03c250d817c6 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 27 Oct 2012 21:32:50 +0200 Subject: [PATCH 019/302] Added optional kwargs to get_expiry_age/date. This change allows for cleaner tests: we can test the exact output. Refs #18194: this change makes it possible to compute session expiry dates at times other than when the session is saved. Fixed #18458: the existence of the `modification` kwarg implies that you must pass it to get_expiry_age/date if you call these functions outside of a short request - response cycle (the intended use case). --- django/contrib/sessions/backends/base.py | 40 ++++++++++++---- django/contrib/sessions/backends/cached_db.py | 2 +- django/contrib/sessions/tests.py | 48 ++++++++++++------- docs/topics/http/sessions.txt | 11 +++++ 4 files changed, 74 insertions(+), 27 deletions(-) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 65bbe059eb..1f63c3b45a 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -170,28 +170,52 @@ class SessionBase(object): _session = property(_get_session) - def get_expiry_age(self, expiry=None): + def get_expiry_age(self, **kwargs): """Get the number of seconds until the session expires. - expiry is an optional parameter specifying the datetime of expiry. + Optionally, this function accepts `modification` and `expiry` keyword + arguments specifying the modification and expiry of the session. """ - if expiry is None: + try: + modification = kwargs['modification'] + except KeyError: + modification = timezone.now() + # Make the difference between "expiry=None passed in kwargs" and + # "expiry not passed in kwargs", in order to guarantee not to trigger + # self.load() when expiry is provided. + try: + expiry = kwargs['expiry'] + except KeyError: expiry = self.get('_session_expiry') + if not expiry: # Checks both None and 0 cases return settings.SESSION_COOKIE_AGE if not isinstance(expiry, datetime): return expiry - delta = expiry - timezone.now() + delta = expiry - modification return delta.days * 86400 + delta.seconds - def get_expiry_date(self): - """Get session the expiry date (as a datetime object).""" - expiry = self.get('_session_expiry') + def get_expiry_date(self, **kwargs): + """Get session the expiry date (as a datetime object). + + Optionally, this function accepts `modification` and `expiry` keyword + arguments specifying the modification and expiry of the session. + """ + try: + modification = kwargs['modification'] + except KeyError: + modification = timezone.now() + # Same comment as in get_expiry_age + try: + expiry = kwargs['expiry'] + except KeyError: + expiry = self.get('_session_expiry') + if isinstance(expiry, datetime): return expiry if not expiry: # Checks both None and 0 cases expiry = settings.SESSION_COOKIE_AGE - return timezone.now() + timedelta(seconds=expiry) + return modification + timedelta(seconds=expiry) def set_expiry(self, value): """ diff --git a/django/contrib/sessions/backends/cached_db.py b/django/contrib/sessions/backends/cached_db.py index 8e4cf8b7e7..31c6fbfce3 100644 --- a/django/contrib/sessions/backends/cached_db.py +++ b/django/contrib/sessions/backends/cached_db.py @@ -40,7 +40,7 @@ class SessionStore(DBStore): ) data = self.decode(s.session_data) cache.set(self.cache_key, data, - self.get_expiry_age(s.expire_date)) + self.get_expiry_age(expiry=s.expire_date)) except (Session.DoesNotExist, SuspiciousOperation): self.create() data = {} diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 527e8eb331..44c3b485e9 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -197,31 +197,43 @@ class SessionTestsMixin(object): self.assertEqual(self.session.get_expiry_age(), settings.SESSION_COOKIE_AGE) def test_custom_expiry_seconds(self): - # Using seconds - self.session.set_expiry(10) - delta = self.session.get_expiry_date() - timezone.now() - self.assertIn(delta.seconds, (9, 10)) + modification = timezone.now() - age = self.session.get_expiry_age() - self.assertIn(age, (9, 10)) + self.session.set_expiry(10) + + date = self.session.get_expiry_date(modification=modification) + self.assertEqual(date, modification + timedelta(seconds=10)) + + age = self.session.get_expiry_age(modification=modification) + self.assertEqual(age, 10) def test_custom_expiry_timedelta(self): - # Using timedelta - self.session.set_expiry(timedelta(seconds=10)) - delta = self.session.get_expiry_date() - timezone.now() - self.assertIn(delta.seconds, (9, 10)) + modification = timezone.now() - age = self.session.get_expiry_age() - self.assertIn(age, (9, 10)) + # Mock timezone.now, because set_expiry calls it on this code path. + original_now = timezone.now + try: + timezone.now = lambda: modification + self.session.set_expiry(timedelta(seconds=10)) + finally: + timezone.now = original_now + + date = self.session.get_expiry_date(modification=modification) + self.assertEqual(date, modification + timedelta(seconds=10)) + + age = self.session.get_expiry_age(modification=modification) + self.assertEqual(age, 10) def test_custom_expiry_datetime(self): - # Using fixed datetime - self.session.set_expiry(timezone.now() + timedelta(seconds=10)) - delta = self.session.get_expiry_date() - timezone.now() - self.assertIn(delta.seconds, (9, 10)) + modification = timezone.now() - age = self.session.get_expiry_age() - self.assertIn(age, (9, 10)) + self.session.set_expiry(modification + timedelta(seconds=10)) + + date = self.session.get_expiry_date(modification=modification) + self.assertEqual(date, modification + timedelta(seconds=10)) + + age = self.session.get_expiry_age(modification=modification) + self.assertEqual(age, 10) def test_custom_expiry_reset(self): self.session.set_expiry(None) diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 0082b75db1..1e043405f4 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -250,12 +250,23 @@ You can edit it multiple times. with no custom expiration (or those set to expire at browser close), this will equal :setting:`SESSION_COOKIE_AGE`. + This function accepts two optional keyword arguments: + + - ``modification``: last modification of the session, as a + :class:`~datetime.datetime` object. Defaults to the current time. + - ``expiry``: expiry information for the session, as a + :class:`~datetime.datetime` object, an :class:`int` (in seconds), or + ``None``. Defaults to the value stored in the session by + :meth:`set_expiry`, if there is one, or ``None``. + .. method:: get_expiry_date Returns the date this session will expire. For sessions with no custom expiration (or those set to expire at browser close), this will equal the date :setting:`SESSION_COOKIE_AGE` seconds from now. + This function accepts the same keyword argumets as :meth:`get_expiry_age`. + .. method:: get_expire_at_browser_close Returns either ``True`` or ``False``, depending on whether the user's From 882c47cd405cfd29194f2e968678a5aa1d6ec75f Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 27 Oct 2012 21:59:35 +0200 Subject: [PATCH 020/302] Improved tests introduced in 04b00b6. These tests are expected to fail for the file session backend because it doesn't handle expiry properly. They didn't because of an error in the test setup sequence. Refs #19200, #18194. --- django/contrib/sessions/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 44c3b485e9..21e8d845b8 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -277,9 +277,11 @@ class SessionTestsMixin(object): try: self.session['foo'] = 'bar' self.session.set_expiry(-timedelta(seconds=10)) - self.session.create() + self.session.save() + old_session_key = self.session.session_key # With an expiry date in the past, the session expires instantly. new_session = self.backend(self.session.session_key) + new_session_key = new_session.session_key self.assertNotIn('foo', new_session) finally: self.session.delete(old_session_key) @@ -353,15 +355,15 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): backend = FileSession def setUp(self): - super(FileSessionTests, self).setUp() # Do file session tests in an isolated directory, and kill it after we're done. self.original_session_file_path = settings.SESSION_FILE_PATH self.temp_session_store = settings.SESSION_FILE_PATH = tempfile.mkdtemp() + super(FileSessionTests, self).setUp() def tearDown(self): + super(FileSessionTests, self).tearDown() settings.SESSION_FILE_PATH = self.original_session_file_path shutil.rmtree(self.temp_session_store) - super(FileSessionTests, self).tearDown() @override_settings( SESSION_FILE_PATH="/if/this/directory/exists/you/have/a/weird/computer") From 5fec97b9df6ea075483276de159e522a29437773 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 27 Oct 2012 23:12:08 +0200 Subject: [PATCH 021/302] Fixed #18194 -- Expiration of file-based sessions * Prevented stale session files from being loaded * Added removal of stale session files in django-admin.py clearsessions Thanks ej for the report, crodjer and Elvard for their inputs. --- django/contrib/sessions/backends/base.py | 12 +++- django/contrib/sessions/backends/cache.py | 4 ++ django/contrib/sessions/backends/db.py | 5 ++ django/contrib/sessions/backends/file.py | 71 +++++++++++++++---- .../sessions/backends/signed_cookies.py | 4 ++ .../management/commands/clearsessions.py | 14 ++-- django/contrib/sessions/tests.py | 61 ++++++++++++++++ docs/ref/django-admin.txt | 2 - docs/topics/http/sessions.txt | 32 ++++++--- 9 files changed, 176 insertions(+), 29 deletions(-) diff --git a/django/contrib/sessions/backends/base.py b/django/contrib/sessions/backends/base.py index 1f63c3b45a..ff8ab7f677 100644 --- a/django/contrib/sessions/backends/base.py +++ b/django/contrib/sessions/backends/base.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import base64 -import time from datetime import datetime, timedelta try: from django.utils.six.moves import cPickle as pickle @@ -309,3 +308,14 @@ class SessionBase(object): Loads the session data and returns a dictionary. """ raise NotImplementedError + + @classmethod + def clear_expired(cls): + """ + Remove expired sessions from the session store. + + If this operation isn't possible on a given backend, it should raise + NotImplementedError. If it isn't necessary, because the backend has + a built-in expiration mechanism, it should be a no-op. + """ + raise NotImplementedError diff --git a/django/contrib/sessions/backends/cache.py b/django/contrib/sessions/backends/cache.py index b66123b915..0c7eb8d2cb 100644 --- a/django/contrib/sessions/backends/cache.py +++ b/django/contrib/sessions/backends/cache.py @@ -65,3 +65,7 @@ class SessionStore(SessionBase): return session_key = self.session_key self._cache.delete(KEY_PREFIX + session_key) + + @classmethod + def clear_expired(cls): + pass diff --git a/django/contrib/sessions/backends/db.py b/django/contrib/sessions/backends/db.py index 4dacc96000..47e89b66e5 100644 --- a/django/contrib/sessions/backends/db.py +++ b/django/contrib/sessions/backends/db.py @@ -71,6 +71,11 @@ class SessionStore(SessionBase): except Session.DoesNotExist: pass + @classmethod + def clear_expired(cls): + Session.objects.filter(expire_date__lt=timezone.now()).delete() + transaction.commit_unless_managed() + # At bottom to avoid circular import from django.contrib.sessions.models import Session diff --git a/django/contrib/sessions/backends/file.py b/django/contrib/sessions/backends/file.py index 20ac2c2087..f3a71f8271 100644 --- a/django/contrib/sessions/backends/file.py +++ b/django/contrib/sessions/backends/file.py @@ -1,3 +1,4 @@ +import datetime import errno import os import tempfile @@ -5,27 +6,36 @@ import tempfile from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation, ImproperlyConfigured - +from django.utils import timezone class SessionStore(SessionBase): """ Implements a file based session store. """ def __init__(self, session_key=None): - self.storage_path = getattr(settings, "SESSION_FILE_PATH", None) - if not self.storage_path: - self.storage_path = tempfile.gettempdir() - - # Make sure the storage path is valid. - if not os.path.isdir(self.storage_path): - raise ImproperlyConfigured( - "The session storage path %r doesn't exist. Please set your" - " SESSION_FILE_PATH setting to an existing directory in which" - " Django can store session data." % self.storage_path) - + self.storage_path = type(self)._get_storage_path() self.file_prefix = settings.SESSION_COOKIE_NAME super(SessionStore, self).__init__(session_key) + @classmethod + def _get_storage_path(cls): + try: + return cls._storage_path + except AttributeError: + storage_path = getattr(settings, "SESSION_FILE_PATH", None) + if not storage_path: + storage_path = tempfile.gettempdir() + + # Make sure the storage path is valid. + if not os.path.isdir(storage_path): + raise ImproperlyConfigured( + "The session storage path %r doesn't exist. Please set your" + " SESSION_FILE_PATH setting to an existing directory in which" + " Django can store session data." % storage_path) + + cls._storage_path = storage_path + return storage_path + VALID_KEY_CHARS = set("abcdef0123456789") def _key_to_file(self, session_key=None): @@ -44,6 +54,18 @@ class SessionStore(SessionBase): return os.path.join(self.storage_path, self.file_prefix + session_key) + def _last_modification(self): + """ + Return the modification time of the file storing the session's content. + """ + modification = os.stat(self._key_to_file()).st_mtime + if settings.USE_TZ: + modification = datetime.datetime.utcfromtimestamp(modification) + modification = modification.replace(tzinfo=timezone.utc) + else: + modification = datetime.datetime.fromtimestamp(modification) + return modification + def load(self): session_data = {} try: @@ -56,6 +78,15 @@ class SessionStore(SessionBase): session_data = self.decode(file_data) except (EOFError, SuspiciousOperation): self.create() + + # Remove expired sessions. + expiry_age = self.get_expiry_age( + modification=self._last_modification(), + expiry=session_data.get('_session_expiry')) + if expiry_age < 0: + session_data = {} + self.delete() + self.create() except IOError: self.create() return session_data @@ -142,3 +173,19 @@ class SessionStore(SessionBase): def clean(self): pass + + @classmethod + def clear_expired(cls): + storage_path = getattr(settings, "SESSION_FILE_PATH", tempfile.gettempdir()) + file_prefix = settings.SESSION_COOKIE_NAME + + for session_file in os.listdir(storage_path): + if not session_file.startswith(file_prefix): + continue + session_key = session_file[len(file_prefix):] + session = cls(session_key) + # When an expired session is loaded, its file is removed, and a + # new file is immediately created. Prevent this by disabling + # the create() method. + session.create = lambda: None + session.load() diff --git a/django/contrib/sessions/backends/signed_cookies.py b/django/contrib/sessions/backends/signed_cookies.py index 23915cf98c..c2b7a3123f 100644 --- a/django/contrib/sessions/backends/signed_cookies.py +++ b/django/contrib/sessions/backends/signed_cookies.py @@ -92,3 +92,7 @@ class SessionStore(SessionBase): return signing.dumps(session_cache, compress=True, salt='django.contrib.sessions.backends.signed_cookies', serializer=PickleSerializer) + + @classmethod + def clear_expired(cls): + pass diff --git a/django/contrib/sessions/management/commands/clearsessions.py b/django/contrib/sessions/management/commands/clearsessions.py index fb7666f85a..8eb23dfee0 100644 --- a/django/contrib/sessions/management/commands/clearsessions.py +++ b/django/contrib/sessions/management/commands/clearsessions.py @@ -1,11 +1,15 @@ +from django.conf import settings from django.core.management.base import NoArgsCommand -from django.utils import timezone +from django.utils.importlib import import_module + class Command(NoArgsCommand): help = "Can be run as a cronjob or directly to clean out expired sessions (only with the database backend at the moment)." def handle_noargs(self, **options): - from django.db import transaction - from django.contrib.sessions.models import Session - Session.objects.filter(expire_date__lt=timezone.now()).delete() - transaction.commit_unless_managed() + engine = import_module(settings.SESSION_ENGINE) + try: + engine.SessionStore.clear_expired() + except NotImplementedError: + self.stderr.write("Session engine '%s' doesn't support clearing " + "expired sessions.\n" % settings.SESSION_ENGINE) diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 21e8d845b8..129cd21735 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -1,4 +1,5 @@ from datetime import timedelta +import os import shutil import string import tempfile @@ -12,6 +13,7 @@ from django.contrib.sessions.backends.file import SessionStore as FileSession from django.contrib.sessions.backends.signed_cookies import SessionStore as CookieSession from django.contrib.sessions.models import Session from django.contrib.sessions.middleware import SessionMiddleware +from django.core import management from django.core.cache import DEFAULT_CACHE_ALIAS from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.http import HttpResponse @@ -319,6 +321,30 @@ class DatabaseSessionTests(SessionTestsMixin, TestCase): del self.session._session_cache self.assertEqual(self.session['y'], 2) + @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") + def test_clearsessions_command(self): + """ + Test clearsessions command for clearing expired sessions. + """ + self.assertEqual(0, Session.objects.count()) + + # One object in the future + self.session['foo'] = 'bar' + self.session.set_expiry(3600) + self.session.save() + + # One object in the past + other_session = self.backend() + other_session['foo'] = 'bar' + other_session.set_expiry(-3600) + other_session.save() + + # Two sessions are in the database before clearsessions... + self.assertEqual(2, Session.objects.count()) + management.call_command('clearsessions') + # ... and one is deleted. + self.assertEqual(1, Session.objects.count()) + @override_settings(USE_TZ=True) class DatabaseSessionWithTimeZoneTests(DatabaseSessionTests): @@ -358,6 +384,9 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): # Do file session tests in an isolated directory, and kill it after we're done. self.original_session_file_path = settings.SESSION_FILE_PATH self.temp_session_store = settings.SESSION_FILE_PATH = tempfile.mkdtemp() + # Reset the file session backend's internal caches + if hasattr(self.backend, '_storage_path'): + del self.backend._storage_path super(FileSessionTests, self).setUp() def tearDown(self): @@ -368,6 +397,7 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): @override_settings( SESSION_FILE_PATH="/if/this/directory/exists/you/have/a/weird/computer") def test_configuration_check(self): + del self.backend._storage_path # Make sure the file backend checks for a good storage dir self.assertRaises(ImproperlyConfigured, self.backend) @@ -381,6 +411,37 @@ class FileSessionTests(SessionTestsMixin, unittest.TestCase): self.assertRaises(SuspiciousOperation, self.backend("a/b/c").load) + @override_settings(SESSION_ENGINE="django.contrib.sessions.backends.file") + def test_clearsessions_command(self): + """ + Test clearsessions command for clearing expired sessions. + """ + storage_path = self.backend._get_storage_path() + file_prefix = settings.SESSION_COOKIE_NAME + + def count_sessions(): + return len([session_file for session_file in os.listdir(storage_path) + if session_file.startswith(file_prefix)]) + + self.assertEqual(0, count_sessions()) + + # One object in the future + self.session['foo'] = 'bar' + self.session.set_expiry(3600) + self.session.save() + + # One object in the past + other_session = self.backend() + other_session['foo'] = 'bar' + other_session.set_expiry(-3600) + other_session.save() + + # Two sessions are in the filesystem before clearsessions... + self.assertEqual(2, count_sessions()) + management.call_command('clearsessions') + # ... and one is deleted. + self.assertEqual(1, count_sessions()) + class CacheSessionTests(SessionTestsMixin, unittest.TestCase): diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index 833db0839c..e0b08450e9 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1200,8 +1200,6 @@ clearsessions Can be run as a cron job or directly to clean out expired sessions. -This is only supported by the database backend at the moment. - ``django.contrib.sitemaps`` --------------------------- diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index 1e043405f4..d9c472d092 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -272,6 +272,13 @@ You can edit it multiple times. Returns either ``True`` or ``False``, depending on whether the user's session cookie will expire when the user's Web browser is closed. + .. method:: SessionBase.clear_expired + + .. versionadded:: 1.5 + + Removes expired sessions from the session store. This class method is + called by :djadmin:`clearsessions`. + Session object guidelines ------------------------- @@ -458,22 +465,29 @@ This setting is a global default and can be overwritten at a per-session level by explicitly calling the :meth:`~backends.base.SessionBase.set_expiry` method of ``request.session`` as described above in `using sessions in views`_. -Clearing the session table +Clearing the session store ========================== -If you're using the database backend, note that session data can accumulate in -the ``django_session`` database table and Django does *not* provide automatic -purging. Therefore, it's your job to purge expired sessions on a regular basis. +As users create new sessions on your website, session data can accumulate in +your session store. If you're using the database backend, the +``django_session`` database table will grow. If you're using the file backend, +your temporary directory will contain an increasing number of files. -To understand this problem, consider what happens when a user uses a session. +To understand this problem, consider what happens with the database backend. When a user logs in, Django adds a row to the ``django_session`` database table. Django updates this row each time the session data changes. If the user logs out manually, Django deletes the row. But if the user does *not* log out, -the row never gets deleted. +the row never gets deleted. A similar process happens with the file backend. -Django provides a sample clean-up script: ``django-admin.py clearsessions``. -That script deletes any session in the session table whose ``expire_date`` is -in the past -- but your application may have different requirements. +Django does *not* provide automatic purging of expired sessions. Therefore, +it's your job to purge expired sessions on a regular basis. Django provides a +clean-up management command for this purpose: :djadmin:`clearsessions`. It's +recommended to call this command on a regular basis, for example as a daily +cron job. + +Note that the cache backend isn't vulnerable to this problem, because caches +automatically delete stale data. Neither is the cookie backend, because the +session data is stored by the users' browsers. Settings ======== From aff9b2f5662f8a007bb90a427190c0d573a3ba65 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 09:34:05 +0100 Subject: [PATCH 022/302] Fixed #19203 -- Added isolation to a humanize test Thanks lrekucki for the report. --- django/conf/global_settings.py | 2 +- django/contrib/humanize/tests.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index f1cbb22880..27bb072114 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -34,7 +34,7 @@ INTERNAL_IPS = () # systems may support all possibilities). When USE_TZ is True, this is # interpreted as the default user time zone. TIME_ZONE = 'America/Chicago' - +TIME_ZONE = 'Europe/Paris' # If you set this to True, Django will use timezone-aware datetimes. USE_TZ = False diff --git a/django/contrib/humanize/tests.py b/django/contrib/humanize/tests.py index a0f13d3ee9..57ce695185 100644 --- a/django/contrib/humanize/tests.py +++ b/django/contrib/humanize/tests.py @@ -1,6 +1,12 @@ from __future__ import unicode_literals import datetime +try: + import pytz +except ImportError: + pytz = None + +from django.conf import settings from django.contrib.humanize.templatetags import humanize from django.template import Template, Context, defaultfilters from django.test import TestCase @@ -10,6 +16,7 @@ from django.utils.timezone import utc from django.utils import translation from django.utils.translation import ugettext as _ from django.utils import tzinfo +from django.utils.unittest import skipIf # Mock out datetime in some tests so they don't fail occasionally when they @@ -141,6 +148,8 @@ class HumanizeTests(TestCase): # As 24h of difference they will never be the same self.assertNotEqual(naturalday_one, naturalday_two) + @skipIf(settings.TIME_ZONE != "Ameria/Chicago" and pytz is None, + "this test requires pytz when a non-default time zone is set") def test_naturalday_uses_localtime(self): # Regression for #18504 # This is 2012-03-08HT19:30:00-06:00 in Ameria/Chicago @@ -148,7 +157,7 @@ class HumanizeTests(TestCase): orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime try: - with override_settings(USE_TZ=True): + with override_settings(TIME_ZONE="America/Chicago", USE_TZ=True): self.humanize_tester([dt], ['yesterday'], 'naturalday') finally: humanize.datetime = orig_humanize_datetime From 785bf0d5a04d8466091a701b732b0c6f87b548e4 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 12:33:38 +0100 Subject: [PATCH 023/302] Reverted unintentional change in aff9b2f. --- django/conf/global_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 27bb072114..f1cbb22880 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -34,7 +34,7 @@ INTERNAL_IPS = () # systems may support all possibilities). When USE_TZ is True, this is # interpreted as the default user time zone. TIME_ZONE = 'America/Chicago' -TIME_ZONE = 'Europe/Paris' + # If you set this to True, Django will use timezone-aware datetimes. USE_TZ = False From 98032f67c725e257bd3c53374ff0ee22e2c77d7c Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 12:40:10 +0100 Subject: [PATCH 024/302] Fixed #14093 -- Improved error message in the cache session backend. Thanks stumbles for the patch. --- django/contrib/sessions/backends/cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django/contrib/sessions/backends/cache.py b/django/contrib/sessions/backends/cache.py index 0c7eb8d2cb..1b4906f923 100644 --- a/django/contrib/sessions/backends/cache.py +++ b/django/contrib/sessions/backends/cache.py @@ -43,7 +43,9 @@ class SessionStore(SessionBase): continue self.modified = True return - raise RuntimeError("Unable to create a new session key.") + raise RuntimeError( + "Unable to create a new session key. " + "It is likely that the cache is unavailable.") def save(self, must_create=False): if must_create: From 611c4d6f1c24763e5e6e331a5dcf9b610288aaa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sun, 28 Oct 2012 16:47:07 +0200 Subject: [PATCH 025/302] Fixed #18823 -- Ensured m2m.clear() works when using through+to_field There was a potential data-loss issue involved -- when clearing instance's m2m assignments it was possible some other instance's m2m data was deleted instead. This commit also improved None handling for to_field cases. --- django/db/models/fields/related.py | 45 ++++++++-- .../m2m_through_regress/models.py | 8 +- .../m2m_through_regress/tests.py | 90 ++++++++++++++++++- 3 files changed, 128 insertions(+), 15 deletions(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index dd9fef34d5..80c62f85c4 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -573,9 +573,31 @@ def create_many_related_manager(superclass, rel): self.reverse = reverse self.through = through self.prefetch_cache_name = prefetch_cache_name - self._pk_val = self.instance.pk - if self._pk_val is None: - raise ValueError("%r instance needs to have a primary key value before a many-to-many relationship can be used." % instance.__class__.__name__) + self._fk_val = self._get_fk_val(instance, source_field_name) + if self._fk_val is None: + raise ValueError('"%r" needs to have a value for field "%s" before ' + 'this many-to-many relationship can be used.' % + (instance, source_field_name)) + # Even if this relation is not to pk, we require still pk value. + # The wish is that the instance has been already saved to DB, + # although having a pk value isn't a guarantee of that. + if instance.pk is None: + raise ValueError("%r instance needs to have a primary key value before " + "a many-to-many relationship can be used." % + instance.__class__.__name__) + + + def _get_fk_val(self, obj, field_name): + """ + Returns the correct value for this relationship's foreign key. This + might be something else than pk value when to_field is used. + """ + fk = self.through._meta.get_field(field_name) + if fk.rel.field_name and fk.rel.field_name != fk.rel.to._meta.pk.attname: + attname = fk.rel.get_related_field().get_attname() + return fk.get_prep_lookup('exact', getattr(obj, attname)) + else: + return obj.pk def get_query_set(self): try: @@ -677,7 +699,11 @@ def create_many_related_manager(superclass, rel): if not router.allow_relation(obj, self.instance): raise ValueError('Cannot add "%r": instance is on database "%s", value is on database "%s"' % (obj, self.instance._state.db, obj._state.db)) - new_ids.add(obj.pk) + fk_val = self._get_fk_val(obj, target_field_name) + if fk_val is None: + raise ValueError('Cannot add "%r": the value for field "%s" is None' % + (obj, target_field_name)) + new_ids.add(self._get_fk_val(obj, target_field_name)) elif isinstance(obj, Model): raise TypeError("'%s' instance expected, got %r" % (self.model._meta.object_name, obj)) else: @@ -685,7 +711,7 @@ def create_many_related_manager(superclass, rel): db = router.db_for_write(self.through, instance=self.instance) vals = self.through._default_manager.using(db).values_list(target_field_name, flat=True) vals = vals.filter(**{ - source_field_name: self._pk_val, + source_field_name: self._fk_val, '%s__in' % target_field_name: new_ids, }) new_ids = new_ids - set(vals) @@ -699,11 +725,12 @@ def create_many_related_manager(superclass, rel): # Add the ones that aren't there already self.through._default_manager.using(db).bulk_create([ self.through(**{ - '%s_id' % source_field_name: self._pk_val, + '%s_id' % source_field_name: self._fk_val, '%s_id' % target_field_name: obj_id, }) for obj_id in new_ids ]) + if self.reverse or source_field_name == self.source_field_name: # Don't send the signal when we are inserting the # duplicate data row for symmetrical reverse entries. @@ -722,7 +749,7 @@ def create_many_related_manager(superclass, rel): old_ids = set() for obj in objs: if isinstance(obj, self.model): - old_ids.add(obj.pk) + old_ids.add(self._get_fk_val(obj, target_field_name)) else: old_ids.add(obj) # Work out what DB we're operating on @@ -736,7 +763,7 @@ def create_many_related_manager(superclass, rel): model=self.model, pk_set=old_ids, using=db) # Remove the specified objects from the join table self.through._default_manager.using(db).filter(**{ - source_field_name: self._pk_val, + source_field_name: self._fk_val, '%s__in' % target_field_name: old_ids }).delete() if self.reverse or source_field_name == self.source_field_name: @@ -756,7 +783,7 @@ def create_many_related_manager(superclass, rel): instance=self.instance, reverse=self.reverse, model=self.model, pk_set=None, using=db) self.through._default_manager.using(db).filter(**{ - source_field_name: self._pk_val + source_field_name: self._fk_val }).delete() if self.reverse or source_field_name == self.source_field_name: # Don't send the signal when we are clearing the diff --git a/tests/regressiontests/m2m_through_regress/models.py b/tests/regressiontests/m2m_through_regress/models.py index 47c24ed5b2..23d3366f22 100644 --- a/tests/regressiontests/m2m_through_regress/models.py +++ b/tests/regressiontests/m2m_through_regress/models.py @@ -62,18 +62,18 @@ class B(models.Model): # Using to_field on the through model @python_2_unicode_compatible class Car(models.Model): - make = models.CharField(max_length=20, unique=True) + make = models.CharField(max_length=20, unique=True, null=True) drivers = models.ManyToManyField('Driver', through='CarDriver') def __str__(self): - return self.make + return "%s" % self.make @python_2_unicode_compatible class Driver(models.Model): - name = models.CharField(max_length=20, unique=True) + name = models.CharField(max_length=20, unique=True, null=True) def __str__(self): - return self.name + return "%s" % self.name @python_2_unicode_compatible class CarDriver(models.Model): diff --git a/tests/regressiontests/m2m_through_regress/tests.py b/tests/regressiontests/m2m_through_regress/tests.py index 458c194f89..828ec3618c 100644 --- a/tests/regressiontests/m2m_through_regress/tests.py +++ b/tests/regressiontests/m2m_through_regress/tests.py @@ -123,18 +123,104 @@ class ToFieldThroughTests(TestCase): self.car = Car.objects.create(make="Toyota") self.driver = Driver.objects.create(name="Ryan Briscoe") CarDriver.objects.create(car=self.car, driver=self.driver) + # We are testing if wrong objects get deleted due to using wrong + # field value in m2m queries. So, it is essential that the pk + # numberings do not match. + # Create one intentionally unused driver to mix up the autonumbering + self.unused_driver = Driver.objects.create(name="Barney Gumble") + # And two intentionally unused cars. + self.unused_car1 = Car.objects.create(make="Trabant") + self.unused_car2 = Car.objects.create(make="Wartburg") def test_to_field(self): self.assertQuerysetEqual( self.car.drivers.all(), [""] - ) + ) def test_to_field_reverse(self): self.assertQuerysetEqual( self.driver.car_set.all(), [""] - ) + ) + + def test_to_field_clear_reverse(self): + self.driver.car_set.clear() + self.assertQuerysetEqual( + self.driver.car_set.all(),[]) + + def test_to_field_clear(self): + self.car.drivers.clear() + self.assertQuerysetEqual( + self.car.drivers.all(),[]) + + # Low level tests for _add_items and _remove_items. We test these methods + # because .add/.remove aren't available for m2m fields with through, but + # through is the only way to set to_field currently. We do want to make + # sure these methods are ready if the ability to use .add or .remove with + # to_field relations is added some day. + def test_add(self): + self.assertQuerysetEqual( + self.car.drivers.all(), + [""] + ) + # Yikes - barney is going to drive... + self.car.drivers._add_items('car', 'driver', self.unused_driver) + self.assertQuerysetEqual( + self.car.drivers.all(), + ["", ""] + ) + + def test_add_null(self): + nullcar = Car.objects.create(make=None) + with self.assertRaises(ValueError): + nullcar.drivers._add_items('car', 'driver', self.unused_driver) + + def test_add_related_null(self): + nulldriver = Driver.objects.create(name=None) + with self.assertRaises(ValueError): + self.car.drivers._add_items('car', 'driver', nulldriver) + + def test_add_reverse(self): + car2 = Car.objects.create(make="Honda") + self.assertQuerysetEqual( + self.driver.car_set.all(), + [""] + ) + self.driver.car_set._add_items('driver', 'car', car2) + self.assertQuerysetEqual( + self.driver.car_set.all(), + ["", ""] + ) + + def test_add_null_reverse(self): + nullcar = Car.objects.create(make=None) + with self.assertRaises(ValueError): + self.driver.car_set._add_items('driver', 'car', nullcar) + + def test_add_null_reverse_related(self): + nulldriver = Driver.objects.create(name=None) + with self.assertRaises(ValueError): + nulldriver.car_set._add_items('driver', 'car', self.car) + + def test_remove(self): + self.assertQuerysetEqual( + self.car.drivers.all(), + [""] + ) + self.car.drivers._remove_items('car', 'driver', self.driver) + self.assertQuerysetEqual( + self.car.drivers.all(),[]) + + def test_remove_reverse(self): + self.assertQuerysetEqual( + self.driver.car_set.all(), + [""] + ) + self.driver.car_set._remove_items('driver', 'car', self.car) + self.assertQuerysetEqual( + self.driver.car_set.all(),[]) + class ThroughLoadDataTestCase(TestCase): fixtures = ["m2m_through"] From 58a086acfbec833f44cd53e984972250bbb67457 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 16:42:34 +0100 Subject: [PATCH 026/302] Required serializer to use bytes in loads/dumps loads has no way to tell if it should provide text or bytes to the serializer; bytes are more reasonnable for a serialized representation, and are the only option for pickled data. dumps can perform conversions on the value it receives from the serializer; but for consistency it seems better to require bytes too. The current code would cause an exception when loading pickled session data. See next commit. Also fixed a bug when checking for compressed data. --- django/core/signing.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/django/core/signing.py b/django/core/signing.py index 147e54780c..92ab968123 100644 --- a/django/core/signing.py +++ b/django/core/signing.py @@ -97,10 +97,10 @@ class JSONSerializer(object): signing.loads. """ def dumps(self, obj): - return json.dumps(obj, separators=(',', ':')) + return json.dumps(obj, separators=(',', ':')).encode('latin-1') def loads(self, data): - return json.loads(data) + return json.loads(data.decode('latin-1')) def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, compress=False): @@ -116,8 +116,10 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, only valid for a given namespace. Leaving this at the default value or re-using a salt value across different parts of your application without good cause is a security risk. + + The serializer is expected to return a bytestring. """ - data = force_bytes(serializer().dumps(obj)) + data = serializer().dumps(obj) # Flag for if it's been compressed or not is_compressed = False @@ -136,20 +138,22 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer, def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None): """ - Reverse of dumps(), raises BadSignature if signature fails + Reverse of dumps(), raises BadSignature if signature fails. + + The serializer is expected to accept a bytestring. """ # TimestampSigner.unsign always returns unicode but base64 and zlib # compression operate on bytes. base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age)) decompress = False - if base64d[0] == b'.': + if base64d[:1] == b'.': # It's compressed; uncompress it first base64d = base64d[1:] decompress = True data = b64_decode(base64d) if decompress: data = zlib.decompress(data) - return serializer().loads(force_str(data)) + return serializer().loads(data) class Signer(object): From 58337b32236eb57d82bf62ed077add3ec69e37f2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 16:51:51 +0100 Subject: [PATCH 027/302] Marked cookies-based session expiry test as an expected failure. Refs #19201. --- django/contrib/sessions/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 129cd21735..718967791d 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -542,3 +542,8 @@ class CookieSessionTests(SessionTestsMixin, TestCase): testing for this behavior is meaningless. """ pass + + @unittest.expectedFailure + def test_actual_expiry(self): + # The cookie backend doesn't handle non-default expiry dates, see #19201 + super(CookieSessionTests, self).test_actual_expiry() From b4420d96023b9d9067ebc8e8dcd8ca7cea41aff1 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 19:56:19 +0100 Subject: [PATCH 028/302] Fixed #18964 -- floatformat test passes under py3k Thanks Russell for the report. --- tests/regressiontests/defaultfilters/tests.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/regressiontests/defaultfilters/tests.py b/tests/regressiontests/defaultfilters/tests.py index bdf3c867c6..d00203e304 100644 --- a/tests/regressiontests/defaultfilters/tests.py +++ b/tests/regressiontests/defaultfilters/tests.py @@ -80,13 +80,16 @@ class DefaultFiltersTests(TestCase): decimal_ctx.prec = old_prec - # This fails because of Python's float handling. Floats with many zeroes - # after the decimal point should be passed in as another type such as - # unicode or Decimal. - @unittest.expectedFailure - def test_floatformat_fail(self): + def test_floatformat_py2_fail(self): self.assertEqual(floatformat(1.00000000000000015, 16), '1.0000000000000002') + # The test above fails because of Python 2's float handling. Floats with + # many zeroes after the decimal point should be passed in as another type + # such as unicode or Decimal. + if not six.PY3: + test_floatformat_py2_fail = unittest.expectedFailure(test_floatformat_py2_fail) + + def test_addslashes(self): self.assertEqual(addslashes('"double quotes" and \'single quotes\''), '\\"double quotes\\" and \\\'single quotes\\\'') From 0b98ef632147a26f2430a3ede48d9e58983cc3ae Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Sun, 28 Oct 2012 18:18:09 -0300 Subject: [PATCH 029/302] Ensure that version detection in docs from 373df56d uses the right Django copy. --- docs/conf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6dd84cffba..ced3fef5f7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,12 +14,15 @@ from __future__ import unicode_literals import sys -import os +from os.path import abspath, dirname, join + +# Make sure we use this copy of Django +sys.path.insert(1, abspath(dirname(dirname(__file__)))) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "_ext"))) +sys.path.append(abspath(join(dirname(__file__), "_ext"))) # -- General configuration ----------------------------------------------------- From effe96b303aba45a88e1a24f613fef9ad974a53a Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 28 Oct 2012 22:35:01 +0100 Subject: [PATCH 030/302] Fixed a typo in aff9b2f. Thanks void. --- django/contrib/humanize/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/contrib/humanize/tests.py b/django/contrib/humanize/tests.py index 57ce695185..c648f544d7 100644 --- a/django/contrib/humanize/tests.py +++ b/django/contrib/humanize/tests.py @@ -148,11 +148,11 @@ class HumanizeTests(TestCase): # As 24h of difference they will never be the same self.assertNotEqual(naturalday_one, naturalday_two) - @skipIf(settings.TIME_ZONE != "Ameria/Chicago" and pytz is None, + @skipIf(settings.TIME_ZONE != "America/Chicago" and pytz is None, "this test requires pytz when a non-default time zone is set") def test_naturalday_uses_localtime(self): # Regression for #18504 - # This is 2012-03-08HT19:30:00-06:00 in Ameria/Chicago + # This is 2012-03-08HT19:30:00-06:00 in America/Chicago dt = datetime.datetime(2012, 3, 9, 1, 30, tzinfo=utc) orig_humanize_datetime, humanize.datetime = humanize.datetime, MockDateTime From ee96f83ab0242476d36321efa7edf0bc17271994 Mon Sep 17 00:00:00 2001 From: Chris McDonald Date: Sun, 28 Oct 2012 14:46:09 -0700 Subject: [PATCH 031/302] Consistently indent comments in project template Makes this file slightly more pep8 compliant. --- django/conf/project_template/project_name/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/conf/project_template/project_name/settings.py b/django/conf/project_template/project_name/settings.py index 6bdaa34988..559e27ca16 100644 --- a/django/conf/project_template/project_name/settings.py +++ b/django/conf/project_template/project_name/settings.py @@ -75,7 +75,7 @@ STATICFILES_DIRS = ( STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', -# 'django.contrib.staticfiles.finders.DefaultStorageFinder', + # 'django.contrib.staticfiles.finders.DefaultStorageFinder', ) # Make this unique, and don't share it with anybody. @@ -85,7 +85,7 @@ SECRET_KEY = '{{ secret_key }}' TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', -# 'django.template.loaders.eggs.Loader', + # 'django.template.loaders.eggs.Loader', ) MIDDLEWARE_CLASSES = ( From f1cc2be0c53858b673afc3b26347d3bb25e424f6 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sun, 28 Oct 2012 23:02:41 +0100 Subject: [PATCH 032/302] Fixed #18575 -- Empty DATABASES should default to dummy backend Thanks delormemarco@gmail.com for the report. --- django/conf/global_settings.py | 8 ++------ django/db/__init__.py | 2 +- django/db/utils.py | 9 ++++++++- tests/regressiontests/backends/tests.py | 11 +++++++++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index f1cbb22880..84296c7493 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -150,12 +150,8 @@ SERVER_EMAIL = 'root@localhost' # Whether to send broken-link emails. SEND_BROKEN_LINK_EMAILS = False -# Database connection info. -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.dummy', - }, -} +# Database connection info. If left empty, will default to the dummy backend. +DATABASES = {} # Classes used to implement DB routing behavior. DATABASE_ROUTERS = [] diff --git a/django/db/__init__.py b/django/db/__init__.py index 26c7add0af..b1980488df 100644 --- a/django/db/__init__.py +++ b/django/db/__init__.py @@ -8,7 +8,7 @@ __all__ = ('backend', 'connection', 'connections', 'router', 'DatabaseError', 'IntegrityError', 'DEFAULT_DB_ALIAS') -if DEFAULT_DB_ALIAS not in settings.DATABASES: +if settings.DATABASES and DEFAULT_DB_ALIAS not in settings.DATABASES: raise ImproperlyConfigured("You must define a '%s' database" % DEFAULT_DB_ALIAS) connections = ConnectionHandler(settings.DATABASES) diff --git a/django/db/utils.py b/django/db/utils.py index 5fa78fe350..a91298626b 100644 --- a/django/db/utils.py +++ b/django/db/utils.py @@ -53,7 +53,14 @@ class ConnectionDoesNotExist(Exception): class ConnectionHandler(object): def __init__(self, databases): - self.databases = databases + if not databases: + self.databases = { + DEFAULT_DB_ALIAS: { + 'ENGINE': 'django.db.backends.dummy', + }, + } + else: + self.databases = databases self._connections = local() def ensure_defaults(self, alias): diff --git a/tests/regressiontests/backends/tests.py b/tests/regressiontests/backends/tests.py index 4766dcf44e..14b3e0053c 100644 --- a/tests/regressiontests/backends/tests.py +++ b/tests/regressiontests/backends/tests.py @@ -23,6 +23,17 @@ from django.utils import unittest from . import models +class DummyBackendTest(TestCase): + def test_no_databases(self): + """ + Test that empty DATABASES setting default to the dummy backend. + """ + DATABASES = {} + conns = ConnectionHandler(DATABASES) + self.assertEqual(conns[DEFAULT_DB_ALIAS].settings_dict['ENGINE'], + 'django.db.backends.dummy') + + class OracleChecks(unittest.TestCase): @unittest.skipUnless(connection.vendor == 'oracle', From 4ea8105120121c7ef0d3dd6eb23f2bf5f55b496a Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Thu, 4 Oct 2012 10:14:50 -0700 Subject: [PATCH 033/302] Fixed #19061 -- added is_active attribute to AbstractBaseUser --- django/contrib/auth/forms.py | 6 +++-- django/contrib/auth/models.py | 2 ++ django/contrib/auth/tests/custom_user.py | 17 +++++++++++++ django/contrib/auth/tests/models.py | 32 ++++++++++++++++++++++++ docs/topics/auth.txt | 9 +++++++ 5 files changed, 64 insertions(+), 2 deletions(-) diff --git a/django/contrib/auth/forms.py b/django/contrib/auth/forms.py index 423e3429e6..9279c52675 100644 --- a/django/contrib/auth/forms.py +++ b/django/contrib/auth/forms.py @@ -209,10 +209,12 @@ class PasswordResetForm(forms.Form): """ UserModel = get_user_model() email = self.cleaned_data["email"] - self.users_cache = UserModel.objects.filter(email__iexact=email, - is_active=True) + self.users_cache = UserModel.objects.filter(email__iexact=email) if not len(self.users_cache): raise forms.ValidationError(self.error_messages['unknown']) + if not any(user.is_active for user in self.users_cache): + # none of the filtered users are active + raise forms.ValidationError(self.error_messages['unknown']) if any((user.password == UNUSABLE_PASSWORD) for user in self.users_cache): raise forms.ValidationError(self.error_messages['unusable']) diff --git a/django/contrib/auth/models.py b/django/contrib/auth/models.py index bd7bf4a162..de4de7cdfd 100644 --- a/django/contrib/auth/models.py +++ b/django/contrib/auth/models.py @@ -232,6 +232,8 @@ class AbstractBaseUser(models.Model): password = models.CharField(_('password'), max_length=128) last_login = models.DateTimeField(_('last login'), default=timezone.now) + is_active = True + REQUIRED_FIELDS = [] class Meta: diff --git a/django/contrib/auth/tests/custom_user.py b/django/contrib/auth/tests/custom_user.py index a29ed6a104..710ac8bdd7 100644 --- a/django/contrib/auth/tests/custom_user.py +++ b/django/contrib/auth/tests/custom_user.py @@ -88,3 +88,20 @@ class ExtensionUser(AbstractUser): class Meta: app_label = 'auth' + + +class IsActiveTestUser1(AbstractBaseUser): + """ + This test user class and derivatives test the default is_active behavior + """ + username = models.CharField(max_length=30, unique=True) + + objects = BaseUserManager() + + USERNAME_FIELD = 'username' + + class Meta: + app_label = 'auth' + + # the is_active attr is provided by AbstractBaseUser + diff --git a/django/contrib/auth/tests/models.py b/django/contrib/auth/tests/models.py index 252a0887c8..cb7d8888fe 100644 --- a/django/contrib/auth/tests/models.py +++ b/django/contrib/auth/tests/models.py @@ -1,4 +1,5 @@ from django.conf import settings +from django.contrib.auth import get_user_model from django.contrib.auth.models import (Group, User, SiteProfileNotAvailable, UserManager) from django.contrib.auth.tests.utils import skipIfCustomUser @@ -98,3 +99,34 @@ class UserManagerTestCase(TestCase): self.assertRaisesMessage(ValueError, 'The given username must be set', User.objects.create_user, username='') + +class IsActiveTestCase(TestCase): + """ + Tests the behavior of the guaranteed is_active attribute + """ + + def test_builtin_user_isactive(self): + user = User.objects.create(username='foo', email='foo@bar.com') + # is_active is true by default + self.assertEqual(user.is_active, True) + user.is_active = False + user.save() + user_fetched = User.objects.get(pk=user.pk) + # the is_active flag is saved + self.assertFalse(user_fetched.is_active) + + @override_settings(AUTH_USER_MODEL='auth.IsActiveTestUser1') + def test_is_active_field_default(self): + """ + tests that the default value for is_active is provided + """ + UserModel = get_user_model() + user = UserModel(username='foo') + self.assertEqual(user.is_active, True) + # you can set the attribute - but it will not save + user.is_active = False + # there should be no problem saving - but the attribute is not saved + user.save() + user_fetched = UserModel.objects.get(pk=user.pk) + # the attribute is always true for newly retrieved instance + self.assertEqual(user_fetched.is_active, True) diff --git a/docs/topics/auth.txt b/docs/topics/auth.txt index d261a3c90b..f7a9084008 100644 --- a/docs/topics/auth.txt +++ b/docs/topics/auth.txt @@ -1911,6 +1911,15 @@ password resets. You must then provide some key implementation details: ``REQUIRED_FIELDS`` must contain all required fields on your User model, but should *not* contain the ``USERNAME_FIELD``. + .. attribute:: User.is_active + + A boolean attribute that indicates whether the user is considered + "active". This attribute is provided as an attribute on + ``AbstractBaseUser`` defaulting to ``True``. How you choose to + implement it will depend on the details of your chosen auth backends. + See the documentation of the :attr:`attribute on the builtin user model + ` for details. + .. method:: User.get_full_name(): A longer formal identifier for the user. A common interpretation From 4c4d08502cbef6e0ed85470b2a5aade7fafa099f Mon Sep 17 00:00:00 2001 From: Luke Plant Date: Mon, 29 Oct 2012 13:40:32 +0000 Subject: [PATCH 034/302] Fixed #17991 - prefetch_related fails with GenericRelation and varchar ID field Thanks to okke@formsma.nl for the report, and carmandrew@gmail.com for the tests. --- django/contrib/contenttypes/generic.py | 6 ++++-- tests/modeltests/prefetch_related/models.py | 10 +++++++++- tests/modeltests/prefetch_related/tests.py | 10 ++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index 29e93eefe7..726f4aa150 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals from collections import defaultdict from functools import partial -from operator import attrgetter from django.core.exceptions import ObjectDoesNotExist from django.db import connection @@ -329,8 +328,11 @@ def create_generic_related_manager(superclass): set(obj._get_pk_val() for obj in instances) } qs = super(GenericRelatedObjectManager, self).get_query_set().using(db).filter(**query) + # We (possibly) need to convert object IDs to the type of the + # instances' PK in order to match up instances: + object_id_converter = instances[0]._meta.pk.to_python return (qs, - attrgetter(self.object_id_field_name), + lambda relobj: object_id_converter(getattr(relobj, self.object_id_field_name)), lambda obj: obj._get_pk_val(), False, self.prefetch_cache_name) diff --git a/tests/modeltests/prefetch_related/models.py b/tests/modeltests/prefetch_related/models.py index 85488f0879..e58997d200 100644 --- a/tests/modeltests/prefetch_related/models.py +++ b/tests/modeltests/prefetch_related/models.py @@ -125,6 +125,10 @@ class TaggedItem(models.Model): related_name='taggeditem_set3') created_by_fkey = models.PositiveIntegerField(null=True) created_by = generic.GenericForeignKey('created_by_ct', 'created_by_fkey',) + favorite_ct = models.ForeignKey(ContentType, null=True, + related_name='taggeditem_set4') + favorite_fkey = models.CharField(max_length=64, null=True) + favorite = generic.GenericForeignKey('favorite_ct', 'favorite_fkey') def __str__(self): return self.tag @@ -132,7 +136,11 @@ class TaggedItem(models.Model): class Bookmark(models.Model): url = models.URLField() - tags = generic.GenericRelation(TaggedItem) + tags = generic.GenericRelation(TaggedItem, related_name='bookmarks') + favorite_tags = generic.GenericRelation(TaggedItem, + content_type_field='favorite_ct', + object_id_field='favorite_fkey', + related_name='favorite_bookmarks') class Comment(models.Model): diff --git a/tests/modeltests/prefetch_related/tests.py b/tests/modeltests/prefetch_related/tests.py index 614a5fc1d6..e81560f01f 100644 --- a/tests/modeltests/prefetch_related/tests.py +++ b/tests/modeltests/prefetch_related/tests.py @@ -319,6 +319,16 @@ class GenericRelationTests(TestCase): for t in b.tags.all()] self.assertEqual(sorted(tags), ["django", "python"]) + def test_charfield_GFK(self): + b = Bookmark.objects.create(url='http://www.djangoproject.com/') + t1 = TaggedItem.objects.create(content_object=b, tag='django') + t2 = TaggedItem.objects.create(content_object=b, favorite=b, tag='python') + + with self.assertNumQueries(3): + bookmark = Bookmark.objects.filter(pk=b.pk).prefetch_related('tags', 'favorite_tags')[0] + self.assertEqual(sorted([i.tag for i in bookmark.tags.all()]), ["django", "python"]) + self.assertEqual([i.tag for i in bookmark.favorite_tags.all()], ["python"]) + class MultiTableInheritanceTest(TestCase): From b774c5993cf80000966ae8f04c985116f98ee5ac Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 29 Oct 2012 17:26:10 +0100 Subject: [PATCH 035/302] Fixed #19172 -- Isolated poisoned_http_host tests from 500 handlers Thanks bernardofontes for the report. --- django/contrib/auth/tests/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django/contrib/auth/tests/views.py b/django/contrib/auth/tests/views.py index bb17576d31..b97d4a7cdf 100644 --- a/django/contrib/auth/tests/views.py +++ b/django/contrib/auth/tests/views.py @@ -115,6 +115,8 @@ class PasswordResetTest(AuthViewsTestCase): self.assertTrue("http://adminsite.com" in mail.outbox[0].body) self.assertEqual(settings.DEFAULT_FROM_EMAIL, mail.outbox[0].from_email) + # Skip any 500 handler action (like sending more mail...) + @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True) def test_poisoned_http_host(self): "Poisoned HTTP_HOST headers can't be used for reset emails" # This attack is based on the way browsers handle URLs. The colon @@ -131,6 +133,8 @@ class PasswordResetTest(AuthViewsTestCase): ) self.assertEqual(len(mail.outbox), 0) + # Skip any 500 handler action (like sending more mail...) + @override_settings(DEBUG_PROPAGATE_EXCEPTIONS=True) def test_poisoned_http_host_admin_site(self): "Poisoned HTTP_HOST headers can't be used for reset emails on admin views" with self.assertRaises(SuspiciousOperation): From d30516e163c1870b2467c63f97c8263f9da40226 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 29 Oct 2012 19:08:07 +0100 Subject: [PATCH 036/302] Prevented leftover files and dirs in admin_scripts tests --- tests/regressiontests/admin_scripts/tests.py | 50 +++++++++----------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/tests/regressiontests/admin_scripts/tests.py b/tests/regressiontests/admin_scripts/tests.py index 3bb8bb0b50..a26d7a6eaa 100644 --- a/tests/regressiontests/admin_scripts/tests.py +++ b/tests/regressiontests/admin_scripts/tests.py @@ -138,6 +138,12 @@ class AdminScriptTestCase(unittest.TestCase): return self.run_test(os.path.join(bin_dir, 'django-admin.py'), args, settings_file) def run_manage(self, args, settings_file=None): + def safe_remove(path): + try: + os.remove(path) + except OSError: + pass + conf_dir = os.path.dirname(conf.__file__) template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py') @@ -150,13 +156,9 @@ class AdminScriptTestCase(unittest.TestCase): "{{ project_name }}", "regressiontests") with open(test_manage_py, 'w') as fp: fp.write(manage_py_contents) + self.addCleanup(safe_remove, test_manage_py) - stdout, stderr = self.run_test('./manage.py', args, settings_file) - - # Cleanup - remove the generated manage.py script - os.remove(test_manage_py) - - return stdout, stderr + return self.run_test('./manage.py', args, settings_file) def assertNoOutput(self, stream): "Utility assertion: assert that the given stream is empty" @@ -1410,9 +1412,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure the startproject management command creates a project" args = ['startproject', 'testproject'] testproject_dir = os.path.join(test_dir, 'testproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) @@ -1423,16 +1425,11 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): def test_invalid_project_name(self): "Make sure the startproject management command validates a project name" - - def cleanup(p): - if os.path.exists(p): - shutil.rmtree(p) - args = ['startproject', '7testproject'] testproject_dir = os.path.join(test_dir, '7testproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(cleanup, testproject_dir) self.assertOutput(err, "Error: '7testproject' is not a valid project name. Please make sure the name begins with a letter or underscore.") self.assertFalse(os.path.exists(testproject_dir)) @@ -1441,9 +1438,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): args = ['startproject', 'testproject', 'othertestproject'] testproject_dir = os.path.join(test_dir, 'othertestproject') os.mkdir(testproject_dir) + self.addCleanup(shutil.rmtree, testproject_dir) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'manage.py'))) @@ -1457,9 +1454,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template') args = ['startproject', '--template', template_path, 'customtestproject'] testproject_dir = os.path.join(test_dir, 'customtestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir'))) @@ -1469,9 +1466,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template' + os.sep) args = ['startproject', '--template', template_path, 'customtestproject'] testproject_dir = os.path.join(test_dir, 'customtestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir'))) @@ -1481,9 +1478,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template.tgz') args = ['startproject', '--template', template_path, 'tarballtestproject'] testproject_dir = os.path.join(test_dir, 'tarballtestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py'))) @@ -1494,9 +1491,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation'] testproject_dir = os.path.join(test_dir, 'altlocation') os.mkdir(testproject_dir) + self.addCleanup(shutil.rmtree, testproject_dir) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py'))) @@ -1507,9 +1504,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): args = ['startproject', '--template', template_url, 'urltestproject'] testproject_dir = os.path.join(test_dir, 'urltestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py'))) @@ -1520,9 +1517,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): args = ['startproject', '--template', template_url, 'urltestproject'] testproject_dir = os.path.join(test_dir, 'urltestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py'))) @@ -1532,9 +1529,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template') args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile'] testproject_dir = os.path.join(test_dir, 'customtestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir'))) @@ -1551,8 +1548,8 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): args = ['startproject', '--template', template_path, 'another_project', 'project_dir'] testproject_dir = os.path.join(test_dir, 'project_dir') os.mkdir(testproject_dir) - out, err = self.run_django_admin(args) self.addCleanup(shutil.rmtree, testproject_dir) + out, err = self.run_django_admin(args) self.assertNoOutput(err) test_manage_py = os.path.join(testproject_dir, 'manage.py') with open(test_manage_py, 'r') as fp: @@ -1564,19 +1561,18 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): "Make sure template context variables are not html escaped" # We're using a custom command so we need the alternate settings self.write_settings('alternate_settings.py') + self.addCleanup(self.remove_settings, 'alternate_settings.py') template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template') args = ['custom_startproject', '--template', template_path, 'another_project', 'project_dir', '--extra', '<&>', '--settings=alternate_settings'] testproject_dir = os.path.join(test_dir, 'project_dir') os.mkdir(testproject_dir) - out, err = self.run_manage(args) self.addCleanup(shutil.rmtree, testproject_dir) + out, err = self.run_manage(args) self.assertNoOutput(err) test_manage_py = os.path.join(testproject_dir, 'additional_dir', 'extra.py') with open(test_manage_py, 'r') as fp: content = fp.read() self.assertIn("<&>", content) - # tidy up alternate settings - self.remove_settings('alternate_settings.py') def test_custom_project_destination_missing(self): """ @@ -1596,9 +1592,9 @@ class StartProject(LiveServerTestCase, AdminScriptTestCase): template_path = os.path.join(test_dir, 'admin_scripts', 'custom_templates', 'project_template') args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject'] testproject_dir = os.path.join(test_dir, 'customtestproject') + self.addCleanup(shutil.rmtree, testproject_dir, True) out, err = self.run_django_admin(args) - self.addCleanup(shutil.rmtree, testproject_dir) self.assertNoOutput(err) self.assertTrue(os.path.isdir(testproject_dir)) path = os.path.join(testproject_dir, 'ticket-18091-non-ascii-template.txt') @@ -1612,8 +1608,8 @@ class DiffSettings(AdminScriptTestCase): def test_basic(self): "Runs without error and emits settings diff." self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'}) + self.addCleanup(self.remove_settings, 'settings_to_diff.py') args = ['diffsettings', '--settings=settings_to_diff'] out, err = self.run_manage(args) - self.remove_settings('settings_to_diff.py') self.assertNoOutput(err) self.assertOutput(out, "FOO = 'bar' ###") From bc00075d51cd3e3b5f9a4d7d0f138e0a819adcb9 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 29 Oct 2012 21:39:12 +0100 Subject: [PATCH 037/302] Fixed #19208 -- Docs for mod_wsgi daemon mode Thanks Graham Dumpleton for the patch. --- docs/howto/deployment/wsgi/modwsgi.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/howto/deployment/wsgi/modwsgi.txt b/docs/howto/deployment/wsgi/modwsgi.txt index 7f68485dff..ead04a4643 100644 --- a/docs/howto/deployment/wsgi/modwsgi.txt +++ b/docs/howto/deployment/wsgi/modwsgi.txt @@ -88,12 +88,20 @@ Using mod_wsgi daemon mode ========================== "Daemon mode" is the recommended mode for running mod_wsgi (on non-Windows -platforms). See the `official mod_wsgi documentation`_ for details on setting -up daemon mode. The only change required to the above configuration if you use -daemon mode is that you can't use ``WSGIPythonPath``; instead you should use -the ``python-path`` option to ``WSGIDaemonProcess``, for example:: +platforms). To create the required daemon process group and delegate the +Django instance to run in it, you will need to add appropriate +``WSGIDaemonProcess`` and ``WSGIProcessGroup`` directives. A further change +required to the above configuration if you use daemon mode is that you can't +use ``WSGIPythonPath``; instead you should use the ``python-path`` option to +``WSGIDaemonProcess``, for example:: WSGIDaemonProcess example.com python-path=/path/to/mysite.com:/path/to/venv/lib/python2.7/site-packages + WSGIProcessGroup example.com + +See the official mod_wsgi documentation for `details on setting up daemon +mode`_. + +.. _details on setting up daemon mode: http://code.google.com/p/modwsgi/wiki/QuickConfigurationGuide#Delegation_To_Daemon_Process .. _serving-files: From 24b2aad8e399fdeb4668fe6f4b7b997cf94100ca Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 29 Oct 2012 23:12:20 +0100 Subject: [PATCH 038/302] Fixed #19209 -- Documented |date:"I". Thanks mitar for the report. --- docs/ref/templates/builtins.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 3b8d058fb4..4aa1a990cd 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1226,7 +1226,8 @@ G Hour, 24-hour format without leading ``'0'`` to ``'23'`` h Hour, 12-hour format. ``'01'`` to ``'12'`` H Hour, 24-hour format. ``'00'`` to ``'23'`` i Minutes. ``'00'`` to ``'59'`` -I Not implemented. +I Daylight Savings Time, whether it's ``'1'`` or ``'0'`` + in effect or not. j Day of the month without leading ``'1'`` to ``'31'`` zeros. l Day of the week, textual, long. ``'Friday'`` From 81f5d4a1a7b955afe530d5292726b3a8a93d7038 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 30 Oct 2012 10:27:01 +0800 Subject: [PATCH 039/302] Added some test guards for some recently added auth tests. Refs #19061, #19057. --- django/contrib/auth/tests/handlers.py | 1 + django/contrib/auth/tests/models.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/django/contrib/auth/tests/handlers.py b/django/contrib/auth/tests/handlers.py index a867aae47a..4b36ba3c13 100644 --- a/django/contrib/auth/tests/handlers.py +++ b/django/contrib/auth/tests/handlers.py @@ -17,6 +17,7 @@ class ModWsgiHandlerTestCase(TransactionTestCase): group = Group.objects.create(name='test_group') user1.groups.add(group) + @skipIfCustomUser def test_check_password(self): """ Verify that check_password returns the correct values as per diff --git a/django/contrib/auth/tests/models.py b/django/contrib/auth/tests/models.py index cb7d8888fe..ca65dee71b 100644 --- a/django/contrib/auth/tests/models.py +++ b/django/contrib/auth/tests/models.py @@ -100,11 +100,13 @@ class UserManagerTestCase(TestCase): 'The given username must be set', User.objects.create_user, username='') + class IsActiveTestCase(TestCase): """ Tests the behavior of the guaranteed is_active attribute """ + @skipIfCustomUser def test_builtin_user_isactive(self): user = User.objects.create(username='foo', email='foo@bar.com') # is_active is true by default From 2b5f848207b1dab35afd6f63d0107629c76d4d9a Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Tue, 2 Oct 2012 09:16:37 -0700 Subject: [PATCH 040/302] Fixed #19057 (again) -- added additional tests --- django/contrib/auth/handlers/modwsgi.py | 7 +---- django/contrib/auth/tests/handlers.py | 42 ++++++++++++++++++------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/django/contrib/auth/handlers/modwsgi.py b/django/contrib/auth/handlers/modwsgi.py index 3229c6714b..5ee4d609f7 100644 --- a/django/contrib/auth/handlers/modwsgi.py +++ b/django/contrib/auth/handlers/modwsgi.py @@ -21,17 +21,12 @@ def check_password(environ, username, password): user = UserModel.objects.get_by_natural_key(username) except UserModel.DoesNotExist: return None - try: - if not user.is_active: - return None - except AttributeError as e: - # a custom user may not support is_active + if not user.is_active: return None return user.check_password(password) finally: db.close_connection() - def groups_for_user(environ, username): """ Authorizes a user based on groups diff --git a/django/contrib/auth/tests/handlers.py b/django/contrib/auth/tests/handlers.py index 4b36ba3c13..04ab46f75b 100644 --- a/django/contrib/auth/tests/handlers.py +++ b/django/contrib/auth/tests/handlers.py @@ -2,31 +2,23 @@ from __future__ import unicode_literals from django.contrib.auth.handlers.modwsgi import check_password, groups_for_user from django.contrib.auth.models import User, Group +from django.contrib.auth.tests import CustomUser from django.contrib.auth.tests.utils import skipIfCustomUser from django.test import TransactionTestCase +from django.test.utils import override_settings class ModWsgiHandlerTestCase(TransactionTestCase): """ Tests for the mod_wsgi authentication handler """ - - def setUp(self): - user1 = User.objects.create_user('test', 'test@example.com', 'test') - User.objects.create_user('test1', 'test1@example.com', 'test1') - group = Group.objects.create(name='test_group') - user1.groups.add(group) - @skipIfCustomUser def test_check_password(self): """ Verify that check_password returns the correct values as per http://code.google.com/p/modwsgi/wiki/AccessControlMechanisms#Apache_Authentication_Provider - - because the custom user available in the test framework does not - support the is_active attribute, we can't test this with a custom - user. """ + User.objects.create_user('test', 'test@example.com', 'test') # User not in database self.assertTrue(check_password({}, 'unknown', '') is None) @@ -34,15 +26,43 @@ class ModWsgiHandlerTestCase(TransactionTestCase): # Valid user with correct password self.assertTrue(check_password({}, 'test', 'test')) + # correct password, but user is inactive + User.objects.filter(username='test').update(is_active=False) + self.assertFalse(check_password({}, 'test', 'test')) + # Valid user with incorrect password self.assertFalse(check_password({}, 'test', 'incorrect')) + @override_settings(AUTH_USER_MODEL='auth.CustomUser') + def test_check_password_custom_user(self): + """ + Verify that check_password returns the correct values as per + http://code.google.com/p/modwsgi/wiki/AccessControlMechanisms#Apache_Authentication_Provider + + with custom user installed + """ + + CustomUser.objects.create_user('test@example.com', '1990-01-01', 'test') + + # User not in database + self.assertTrue(check_password({}, 'unknown', '') is None) + + # Valid user with correct password' + self.assertTrue(check_password({}, 'test@example.com', 'test')) + + # Valid user with incorrect password + self.assertFalse(check_password({}, 'test@example.com', 'incorrect')) + @skipIfCustomUser def test_groups_for_user(self): """ Check that groups_for_user returns correct values as per http://code.google.com/p/modwsgi/wiki/AccessControlMechanisms#Apache_Group_Authorisation """ + user1 = User.objects.create_user('test', 'test@example.com', 'test') + User.objects.create_user('test1', 'test1@example.com', 'test1') + group = Group.objects.create(name='test_group') + user1.groups.add(group) # User not in database self.assertEqual(groups_for_user({}, 'unknown'), []) From 9741912a9aaad083eaa8b9780cde37e1843cc4ae Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sun, 9 Sep 2012 16:25:06 -0400 Subject: [PATCH 041/302] Fixed #17869 - force logout when REMOTE_USER header disappears If the current sessions user was logged in via a remote user backend log out the user if REMOTE_USER header not available - otherwise leave it to other auth middleware to install the AnonymousUser. Thanks to Sylvain Bouchard for the initial patch and ticket maintenance. --- django/contrib/auth/middleware.py | 17 +++++++++++++--- django/contrib/auth/tests/remote_user.py | 25 ++++++++++++++++++++++-- docs/releases/1.5.txt | 3 +++ 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/django/contrib/auth/middleware.py b/django/contrib/auth/middleware.py index 0398cfaf1e..f38efdd1d2 100644 --- a/django/contrib/auth/middleware.py +++ b/django/contrib/auth/middleware.py @@ -1,4 +1,6 @@ from django.contrib import auth +from django.contrib.auth import load_backend +from django.contrib.auth.backends import RemoteUserBackend from django.core.exceptions import ImproperlyConfigured from django.utils.functional import SimpleLazyObject @@ -47,9 +49,18 @@ class RemoteUserMiddleware(object): try: username = request.META[self.header] except KeyError: - # If specified header doesn't exist then return (leaving - # request.user set to AnonymousUser by the - # AuthenticationMiddleware). + # If specified header doesn't exist then remove any existing + # authenticated remote-user, or return (leaving request.user set to + # AnonymousUser by the AuthenticationMiddleware). + if request.user.is_authenticated(): + try: + stored_backend = load_backend(request.session.get( + auth.BACKEND_SESSION_KEY, '')) + if isinstance(stored_backend, RemoteUserBackend): + auth.logout(request) + except ImproperlyConfigured as e: + # backend failed to load + auth.logout(request) return # If the user is already authenticated and that user is the user we are # getting passed in the headers, then the correct user is already diff --git a/django/contrib/auth/tests/remote_user.py b/django/contrib/auth/tests/remote_user.py index 9b0f6f8be3..0e59b291a8 100644 --- a/django/contrib/auth/tests/remote_user.py +++ b/django/contrib/auth/tests/remote_user.py @@ -1,8 +1,9 @@ from datetime import datetime from django.conf import settings +from django.contrib.auth import authenticate from django.contrib.auth.backends import RemoteUserBackend -from django.contrib.auth.models import User +from django.contrib.auth.models import User, AnonymousUser from django.contrib.auth.tests.utils import skipIfCustomUser from django.test import TestCase from django.utils import timezone @@ -23,7 +24,7 @@ class RemoteUserTest(TestCase): self.curr_middleware = settings.MIDDLEWARE_CLASSES self.curr_auth = settings.AUTHENTICATION_BACKENDS settings.MIDDLEWARE_CLASSES += (self.middleware,) - settings.AUTHENTICATION_BACKENDS = (self.backend,) + settings.AUTHENTICATION_BACKENDS += (self.backend,) def test_no_remote_user(self): """ @@ -97,6 +98,26 @@ class RemoteUserTest(TestCase): response = self.client.get('/remote_user/', REMOTE_USER=self.known_user) self.assertEqual(default_login, response.context['user'].last_login) + def test_header_disappears(self): + """ + Tests that a logged in user is logged out automatically when + the REMOTE_USER header disappears during the same browser session. + """ + User.objects.create(username='knownuser') + # Known user authenticates + response = self.client.get('/remote_user/', REMOTE_USER=self.known_user) + self.assertEqual(response.context['user'].username, 'knownuser') + # During the session, the REMOTE_USER header disappears. Should trigger logout. + response = self.client.get('/remote_user/') + self.assertEqual(response.context['user'].is_anonymous(), True) + # verify the remoteuser middleware will not remove a user + # authenticated via another backend + User.objects.create_user(username='modeluser', password='foo') + self.client.login(username='modeluser', password='foo') + authenticate(username='modeluser', password='foo') + response = self.client.get('/remote_user/') + self.assertEqual(response.context['user'].username, 'modeluser') + def tearDown(self): """Restores settings to avoid breaking other tests.""" settings.MIDDLEWARE_CLASSES = self.curr_middleware diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index ebf88e83b9..3ee1b2d21f 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -296,6 +296,9 @@ Django 1.5 also includes several smaller improvements worth noting: you to test equality for XML content at a semantic level, without caring for syntax differences (spaces, attribute order, etc.). +* RemoteUserMiddleware now forces logout when the REMOTE_USER header + disappears during the same browser session. + Backwards incompatible changes in 1.5 ===================================== From 6de6988f9990b4b53f5a20bfc3811b3cc49291c5 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 29 Oct 2012 20:33:00 +0100 Subject: [PATCH 042/302] Fixed #5076 -- Properly decode POSTs with non-utf-8 payload encoding Thanks daniel at blogg.se for the report and Aymeric Augustin for his assistance on the patch. --- django/core/handlers/wsgi.py | 24 ++++++++++++++++++++++++ tests/regressiontests/requests/tests.py | 15 +++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py index 45cb2268ed..4c0710549a 100644 --- a/django/core/handlers/wsgi.py +++ b/django/core/handlers/wsgi.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import codecs import logging import sys from io import BytesIO @@ -144,6 +145,14 @@ class WSGIRequest(http.HttpRequest): self.META['PATH_INFO'] = path_info self.META['SCRIPT_NAME'] = script_name self.method = environ['REQUEST_METHOD'].upper() + _, content_params = self._parse_content_type(self.META.get('CONTENT_TYPE', '')) + if 'charset' in content_params: + try: + codecs.lookup(content_params['charset']) + except LookupError: + pass + else: + self.encoding = content_params['charset'] self._post_parse_error = False try: content_length = int(self.environ.get('CONTENT_LENGTH')) @@ -155,6 +164,21 @@ class WSGIRequest(http.HttpRequest): def _is_secure(self): return 'wsgi.url_scheme' in self.environ and self.environ['wsgi.url_scheme'] == 'https' + def _parse_content_type(self, ctype): + """ + Media Types parsing according to RFC 2616, section 3.7. + + Returns the data type and parameters. For example: + Input: "text/plain; charset=iso-8859-1" + Output: ('text/plain', {'charset': 'iso-8859-1'}) + """ + content_type, _, params = ctype.partition(';') + content_params = {} + for parameter in params.split(';'): + k, _, v = parameter.strip().partition('=') + content_params[k] = v + return content_type, content_params + def _get_request(self): if not hasattr(self, '_request'): self._request = datastructures.MergeDict(self.POST, self.GET) diff --git a/tests/regressiontests/requests/tests.py b/tests/regressiontests/requests/tests.py index 6522620d5f..eaf25ea7a6 100644 --- a/tests/regressiontests/requests/tests.py +++ b/tests/regressiontests/requests/tests.py @@ -1,3 +1,4 @@ +# -*- encoding: utf-8 -*- from __future__ import unicode_literals import time @@ -352,6 +353,20 @@ class RequestsTests(unittest.TestCase): self.assertRaises(Exception, lambda: request.body) self.assertEqual(request.POST, {}) + def test_alternate_charset_POST(self): + """ + Test a POST with non-utf-8 payload encoding. + """ + from django.utils.http import urllib_parse + payload = FakePayload(urllib_parse.urlencode({'key': 'España'.encode('latin-1')})) + request = WSGIRequest({ + 'REQUEST_METHOD': 'POST', + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded; charset=iso-8859-1', + 'wsgi.input': payload, + }) + self.assertEqual(request.POST, {'key': ['España']}) + def test_body_after_POST_multipart(self): """ Reading body after parsing multipart is not allowed From 5dc4437dfad8241a5830e4d2ab59635b814281d6 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 30 Oct 2012 09:28:19 +0100 Subject: [PATCH 043/302] Fixed #15714 -- Added note about capitalization of LANG_INFO name_local --- django/conf/locale/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django/conf/locale/__init__.py b/django/conf/locale/__init__.py index 93e98194a4..dcd525fc02 100644 --- a/django/conf/locale/__init__.py +++ b/django/conf/locale/__init__.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +# About name_local: capitalize it as if your language name was appearing +# inside a sentence in your language. + LANG_INFO = { 'ar': { 'bidi': True, @@ -137,7 +140,7 @@ LANG_INFO = { 'bidi': False, 'code': 'fr', 'name': 'French', - 'name_local': 'Fran\xe7ais', + 'name_local': 'fran\xe7ais', }, 'fy-nl': { 'bidi': False, From 43d7cee86e05dc267d66c3a308746fc1ac58271e Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 30 Oct 2012 14:02:54 +0100 Subject: [PATCH 044/302] Added release notes for 1.6. Since 1.5 is feature-frozen, we need them to document new features. --- docs/releases/1.6.txt | 32 ++++++++++++++++++++++++++++++++ docs/releases/index.txt | 7 +++++++ 2 files changed, 39 insertions(+) create mode 100644 docs/releases/1.6.txt diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt new file mode 100644 index 0000000000..ef162a8de3 --- /dev/null +++ b/docs/releases/1.6.txt @@ -0,0 +1,32 @@ +============================================ +Django 1.6 release notes - UNDER DEVELOPMENT +============================================ + +Welcome to Django 1.6! + +These release notes cover the `new features`_, as well as some `backwards +incompatible changes`_ you'll want to be aware of when upgrading from Django +1.5 or older versions. We've also dropped some features, which are detailed in +:doc:`our deprecation plan `, and we've `begun the +deprecation process for some features`_. + +.. _`new features`: `What's new in Django 1.6`_ +.. _`backwards incompatible changes`: `Backwards incompatible changes in 1.6`_ +.. _`begun the deprecation process for some features`: `Features deprecated in 1.6`_ + +What's new in Django 1.6 +======================== + +Backwards incompatible changes in 1.6 +===================================== + +.. warning:: + + In addition to the changes outlined in this section, be sure to review the + :doc:`deprecation plan ` for any features that + have been removed. If you haven't updated your code within the + deprecation timeline for a given feature, its removal may appear as a + backwards incompatible change. + +Features deprecated in 1.6 +========================== diff --git a/docs/releases/index.txt b/docs/releases/index.txt index 6df9821f56..e71376447d 100644 --- a/docs/releases/index.txt +++ b/docs/releases/index.txt @@ -16,6 +16,13 @@ Final releases .. _development_release_notes: +1.6 release +----------- +.. toctree:: + :maxdepth: 1 + + 1.6 + 1.5 release ----------- .. toctree:: From 9a02851340c5c30b8e2c174783cd86d5cab7ab81 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 27 Oct 2012 22:38:15 +0200 Subject: [PATCH 045/302] Fixed #17744 -- Reset default file storage with setting_changed signal --- django/test/signals.py | 7 +++++++ docs/topics/testing.txt | 19 ++++++++++--------- .../staticfiles_tests/tests.py | 9 ++------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/django/test/signals.py b/django/test/signals.py index d140304f1d..a96bdff3b3 100644 --- a/django/test/signals.py +++ b/django/test/signals.py @@ -5,6 +5,7 @@ from django.conf import settings from django.db import connections from django.dispatch import receiver, Signal from django.utils import timezone +from django.utils.functional import empty template_rendered = Signal(providing_args=["template", "context"]) @@ -72,3 +73,9 @@ def language_changed(**kwargs): trans_real._default = None if kwargs['setting'] == 'LOCALE_PATHS': trans_real._translations = {} + +@receiver(setting_changed) +def file_storage_changed(**kwargs): + if kwargs['setting'] in ('MEDIA_ROOT', 'DEFAULT_FILE_STORAGE'): + from django.core.files.storage import default_storage + default_storage._wrapped = empty diff --git a/docs/topics/testing.txt b/docs/topics/testing.txt index 7c25a8b3ff..f5fd4fe3e6 100644 --- a/docs/topics/testing.txt +++ b/docs/topics/testing.txt @@ -1594,15 +1594,16 @@ callbacks to clean up and otherwise reset state when settings are changed. Django itself uses this signal to reset various data: -=========================== ======================== -Overriden settings Data reset -=========================== ======================== -USE_TZ, TIME_ZONE Databases timezone -TEMPLATE_CONTEXT_PROCESSORS Context processors cache -TEMPLATE_LOADERS Template loaders cache -SERIALIZATION_MODULES Serializers cache -LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations -=========================== ======================== +================================ ======================== +Overriden settings Data reset +================================ ======================== +USE_TZ, TIME_ZONE Databases timezone +TEMPLATE_CONTEXT_PROCESSORS Context processors cache +TEMPLATE_LOADERS Template loaders cache +SERIALIZATION_MODULES Serializers cache +LOCALE_PATHS, LANGUAGE_CODE Default translation and loaded translations +MEDIA_ROOT, DEFAULT_FILE_STORAGE Default file storage +================================ ======================== Emptying the test outbox ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/regressiontests/staticfiles_tests/tests.py b/tests/regressiontests/staticfiles_tests/tests.py index 7ecbccc448..69e30613a8 100644 --- a/tests/regressiontests/staticfiles_tests/tests.py +++ b/tests/regressiontests/staticfiles_tests/tests.py @@ -12,7 +12,6 @@ from django.template import loader, Context from django.conf import settings from django.core.cache.backends.base import BaseCache from django.core.exceptions import ImproperlyConfigured -from django.core.files.storage import default_storage from django.core.management import call_command from django.test import TestCase from django.test.utils import override_settings @@ -48,10 +47,9 @@ class BaseStaticFilesTestCase(object): Test case with a couple utility assertions. """ def setUp(self): - # Clear the cached default_storage out, this is because when it first - # gets accessed (by some other test), it evaluates settings.MEDIA_ROOT, + # Clear the cached staticfiles_storage out, this is because when it first + # gets accessed (by some other test), it evaluates settings.STATIC_ROOT, # since we're planning on changing that we need to clear out the cache. - default_storage._wrapped = empty storage.staticfiles_storage._wrapped = empty # Clear the cached staticfile finders, so they are reinitialized every # run and pick up changes in settings.STATICFILES_DIRS. @@ -709,9 +707,6 @@ class TestMiscFinder(TestCase): """ A few misc finder tests. """ - def setUp(self): - default_storage._wrapped = empty - def test_get_finder(self): self.assertIsInstance(finders.get_finder( 'django.contrib.staticfiles.finders.FileSystemFinder'), From 73245b3285f63694d2c63c9b197165dc6d9cfd4e Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 30 Oct 2012 22:20:42 +0100 Subject: [PATCH 046/302] Prevented file_upload tests to leave files behind Refs #19206. --- tests/regressiontests/file_uploads/models.py | 9 +--- tests/regressiontests/file_uploads/tests.py | 49 +++++++++++++------- tests/regressiontests/file_uploads/views.py | 4 +- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/tests/regressiontests/file_uploads/models.py b/tests/regressiontests/file_uploads/models.py index 28e71c44bc..3336cc9d90 100644 --- a/tests/regressiontests/file_uploads/models.py +++ b/tests/regressiontests/file_uploads/models.py @@ -1,12 +1,5 @@ -import tempfile -import os - -from django.core.files.storage import FileSystemStorage from django.db import models -temp_storage = FileSystemStorage(tempfile.mkdtemp()) -UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload') - class FileModel(models.Model): - testfile = models.FileField(storage=temp_storage, upload_to='test_upload') + testfile = models.FileField(upload_to='test_upload') diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py index 918f77d73c..8fa140bc18 100644 --- a/tests/regressiontests/file_uploads/tests.py +++ b/tests/regressiontests/file_uploads/tests.py @@ -7,22 +7,36 @@ import hashlib import json import os import shutil +import tempfile as sys_tempfile from django.core.files import temp as tempfile from django.core.files.uploadedfile import SimpleUploadedFile from django.http.multipartparser import MultiPartParser from django.test import TestCase, client +from django.test.utils import override_settings from django.utils.encoding import force_bytes from django.utils.six import StringIO from django.utils import unittest from . import uploadhandler -from .models import FileModel, temp_storage, UPLOAD_TO +from .models import FileModel UNICODE_FILENAME = 'test-0123456789_中文_Orléans.jpg' +MEDIA_ROOT = sys_tempfile.mkdtemp() +UPLOAD_TO = os.path.join(MEDIA_ROOT, 'test_upload') +@override_settings(MEDIA_ROOT=MEDIA_ROOT) class FileUploadTests(TestCase): + @classmethod + def setUpClass(cls): + if not os.path.isdir(MEDIA_ROOT): + os.makedirs(MEDIA_ROOT) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(MEDIA_ROOT) + def test_simple_upload(self): with open(__file__, 'rb') as fp: post_data = { @@ -83,7 +97,8 @@ class FileUploadTests(TestCase): self.assertEqual(received['file'], test_string) def test_unicode_file_name(self): - tdir = tempfile.gettempdir() + tdir = sys_tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tdir, True) # This file contains chinese symbols and an accented char in the name. with open(os.path.join(tdir, UNICODE_FILENAME), 'w+b') as file1: @@ -96,11 +111,6 @@ class FileUploadTests(TestCase): response = self.client.post('/file_uploads/unicode_name/', post_data) - try: - os.unlink(file1.name) - except OSError: - pass - self.assertEqual(response.status_code, 200) def test_dangerous_file_names(self): @@ -347,26 +357,28 @@ class FileUploadTests(TestCase): # shouldn't differ. self.assertEqual(os.path.basename(obj.testfile.path), 'MiXeD_cAsE.txt') -class DirectoryCreationTests(unittest.TestCase): +@override_settings(MEDIA_ROOT=MEDIA_ROOT) +class DirectoryCreationTests(TestCase): """ Tests for error handling during directory creation via _save_FIELD_file (ticket #6450) """ + @classmethod + def setUpClass(cls): + if not os.path.isdir(MEDIA_ROOT): + os.makedirs(MEDIA_ROOT) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(MEDIA_ROOT) + def setUp(self): self.obj = FileModel() - if not os.path.isdir(temp_storage.location): - os.makedirs(temp_storage.location) - if os.path.isdir(UPLOAD_TO): - os.chmod(UPLOAD_TO, 0o700) - shutil.rmtree(UPLOAD_TO) - - def tearDown(self): - os.chmod(temp_storage.location, 0o700) - shutil.rmtree(temp_storage.location) def test_readonly_root(self): """Permission errors are not swallowed""" - os.chmod(temp_storage.location, 0o500) + os.chmod(MEDIA_ROOT, 0o500) + self.addCleanup(os.chmod, MEDIA_ROOT, 0o700) try: self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', b'x')) except OSError as err: @@ -378,6 +390,7 @@ class DirectoryCreationTests(unittest.TestCase): """The correct IOError is raised when the upload directory name exists but isn't a directory""" # Create a file with the upload directory name open(UPLOAD_TO, 'wb').close() + self.addCleanup(os.remove, UPLOAD_TO) with self.assertRaises(IOError) as exc_info: self.obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', b'x')) # The test needs to be done on a specific string as IOError diff --git a/tests/regressiontests/file_uploads/views.py b/tests/regressiontests/file_uploads/views.py index fcf32cecea..eb7b654c09 100644 --- a/tests/regressiontests/file_uploads/views.py +++ b/tests/regressiontests/file_uploads/views.py @@ -9,8 +9,8 @@ from django.http import HttpResponse, HttpResponseServerError from django.utils import six from django.utils.encoding import force_bytes -from .models import FileModel, UPLOAD_TO -from .tests import UNICODE_FILENAME +from .models import FileModel +from .tests import UNICODE_FILENAME, UPLOAD_TO from .uploadhandler import QuotaUploadHandler, ErroringUploadHandler From 2f035a9723d62a63027df4c779c665e8191dd95b Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 30 Oct 2012 23:05:47 +0100 Subject: [PATCH 047/302] Fixed #19174 -- Fixed capitalization errors in LANG_INFO Thanks waldeinburg for the report. --- django/conf/locale/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django/conf/locale/__init__.py b/django/conf/locale/__init__.py index dcd525fc02..45e56b6d19 100644 --- a/django/conf/locale/__init__.py +++ b/django/conf/locale/__init__.py @@ -56,7 +56,7 @@ LANG_INFO = { 'bidi': False, 'code': 'da', 'name': 'Danish', - 'name_local': 'Dansk', + 'name_local': 'dansk', }, 'de': { 'bidi': False, @@ -272,7 +272,7 @@ LANG_INFO = { 'bidi': False, 'code': 'nb', 'name': 'Norwegian Bokmal', - 'name_local': 'Norsk (bokm\xe5l)', + 'name_local': 'norsk (bokm\xe5l)', }, 'ne': { 'bidi': False, @@ -290,13 +290,13 @@ LANG_INFO = { 'bidi': False, 'code': 'nn', 'name': 'Norwegian Nynorsk', - 'name_local': 'Norsk (nynorsk)', + 'name_local': 'norsk (nynorsk)', }, 'no': { 'bidi': False, 'code': 'no', 'name': 'Norwegian', - 'name_local': 'Norsk', + 'name_local': 'norsk', }, 'pa': { 'bidi': False, @@ -368,7 +368,7 @@ LANG_INFO = { 'bidi': False, 'code': 'sv', 'name': 'Swedish', - 'name_local': 'Svenska', + 'name_local': 'svenska', }, 'sw': { 'bidi': False, From 08cf54990ae112083b159aa4e263c1f64f396f39 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 30 Oct 2012 15:53:56 -0400 Subject: [PATCH 048/302] Fixed #16671 - Added a tutorial on reuseable apps Thank-you Katie Miller and Ben Sturmfels for the initial draft, as well as Russ and Carl for the reviews. --- AUTHORS | 2 + docs/index.txt | 3 + docs/intro/index.txt | 13 +- docs/intro/reusable-apps.txt | 363 +++++++++++++++++++++++++++++++++++ docs/intro/tutorial03.txt | 6 + docs/intro/tutorial04.txt | 9 +- 6 files changed, 388 insertions(+), 8 deletions(-) create mode 100644 docs/intro/reusable-apps.txt diff --git a/AUTHORS b/AUTHORS index 5799b941ff..6f9b410cca 100644 --- a/AUTHORS +++ b/AUTHORS @@ -380,6 +380,7 @@ answer newbie questions, and generally made Django that much better: Christian Metts michal@plovarna.cz Slawek Mikula + Katie Miller Shawn Milochik mitakummaa@gmail.com Taylor Mitchell @@ -510,6 +511,7 @@ answer newbie questions, and generally made Django that much better: Johan C. Stöver Nowell Strite Thomas Stromberg + Ben Sturmfels Travis Swicegood Pascal Varet SuperJared diff --git a/docs/index.txt b/docs/index.txt index 5055edf7e7..a6d9ed2b13 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -46,6 +46,9 @@ Are you new to Django or to programming? This is the place to start! :doc:`Part 3 ` | :doc:`Part 4 ` +* **Advanced Tutorials:** + :doc:`How to write reusable apps ` + The model layer =============== diff --git a/docs/intro/index.txt b/docs/intro/index.txt index 19290a53c6..afb1825b87 100644 --- a/docs/intro/index.txt +++ b/docs/intro/index.txt @@ -6,31 +6,32 @@ place: read this material to quickly get up and running. .. toctree:: :maxdepth: 1 - + overview install tutorial01 tutorial02 tutorial03 tutorial04 + reusable-apps whatsnext - + .. seealso:: If you're new to Python_, you might want to start by getting an idea of what the language is like. Django is 100% Python, so if you've got minimal comfort with Python you'll probably get a lot more out of Django. - + If you're new to programming entirely, you might want to start with this `list of Python resources for non-programmers`_ - + If you already know a few other languages and want to get up to speed with Python quickly, we recommend `Dive Into Python`_ (also available in a `dead-tree version`_). If that's not quite your style, there are quite a few other `books about Python`_. - + .. _python: http://python.org/ .. _list of Python resources for non-programmers: http://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _dive into python: http://diveintopython.net/ .. _dead-tree version: http://www.amazon.com/exec/obidos/ASIN/1590593561/ref=nosim/jacobian20 - .. _books about Python: http://wiki.python.org/moin/PythonBooks \ No newline at end of file + .. _books about Python: http://wiki.python.org/moin/PythonBooks diff --git a/docs/intro/reusable-apps.txt b/docs/intro/reusable-apps.txt new file mode 100644 index 0000000000..200051593c --- /dev/null +++ b/docs/intro/reusable-apps.txt @@ -0,0 +1,363 @@ +============================================= +Advanced tutorial: How to write reusable apps +============================================= + +This advanced tutorial begins where :doc:`Tutorial 4 ` left +off. We'll be turning our Web-poll into a standalone Python package you can +reuse in new projects and share with other people. + +If you haven't recently completed Tutorials 1–4, we encourage you to review +these so that your example project matches the one described below. + +Reusability matters +=================== + +It's a lot of work to design, build, test and maintain a web application. Many +Python and Django projects share common problems. Wouldn't it be great if we +could save some of this repeated work? + +Reusability is the way of life in Python. `The Python Package Index (PyPI) +`_ has a vast +range of packages you can use in your own Python programs. Check out `Django +Packages `_ for existing reusable apps you could +incorporate in your project. Django itself is also just a Python package. This +means that you can take existing Python packages or Django apps and compose +them into your own web project. You only need to write the parts that make +your project unique. + +Let's say you were starting a new project that needed a polls app like the one +we've been working on. How do you make this app reusable? Luckily, you're well +on the way already. In :doc:`Tutorial 3 `, we saw how we +could decouple polls from the project-level URLconf using an ``include``. +In this tutorial, we'll take further steps to make the app easy to use in new +projects and ready to publish for others to install and use. + +.. admonition:: Package? App? + + A Python `package `_ + provides a way of grouping related Python code for easy reuse. A package + contains one or more files of Python code (also known as "modules"). + + A package can be imported with ``import foo.bar`` or ``from foo import + bar``. For a directory (like ``polls``) to form a package, it must contain + a special file ``__init__.py``, even if this file is empty. + + A Django *app* is just a Python package that is specifically intended for + use in a Django project. An app may also use common Django conventions, + such as having a ``models.py`` file. + + Later on we use the term *packaging* to describe the process of making a + Python package easy for others to install. It can be a little confusing, we + know. + +Completing your reusable app +============================ + +After the previous tutorials, our project should look like this:: + + mysite/ + manage.py + mysite/ + __init__.py + settings.py + urls.py + wsgi.py + polls/ + admin.py + __init__.py + models.py + tests.py + urls.py + views.py + +You also have a directory somewhere called ``mytemplates`` which you created in +:doc:`Tutorial 2 `. You specified its location in the +TEMPLATE_DIRS setting. This directory should look like this:: + + mytemplates/ + admin/ + base_site.html + polls/ + detail.html + index.html + results.html + +The polls app is already a Python package, thanks to the ``polls/__init__.py`` +file. That's a great start, but we can't just pick up this package and drop it +into a new project. The polls templates are currently stored in the +project-wide ``mytemplates`` directory. To make the app self-contained, it +should also contain the necessary templates. + +Inside the ``polls`` app, create a new ``templates`` directory. Now move the +``polls`` template directory from ``mytemplates`` into the new +``templates``. Your project should now look like this:: + + mysite/ + manage.py + mysite/ + __init__.py + settings.py + urls.py + wsgi.py + polls/ + admin.py + __init__.py + models.py + templates/ + polls/ + detail.html + index.html + results.html + tests.py + urls.py + views.py + +Your project-wide templates directory should now look like this:: + + mytemplates/ + admin/ + base_site.html + +Looking good! Now would be a good time to confirm that your polls application +still works correctly. How does Django know how to find the new location of +the polls templates even though we didn't modify :setting:`TEMPLATE_DIRS`? +Django has a :setting:`TEMPLATE_LOADERS` setting which contains a list +of callables that know how to import templates from various sources. One of +the defaults is :class:`django.template.loaders.app_directories.Loader` which +looks for a "templates" subdirectory in each of the :setting:`INSTALLED_APPS`. + +The ``polls`` directory could now be copied into a new Django project and +immediately reused. It's not quite ready to be published though. For that, we +need to package the app to make it easy for others to install. + +.. admonition:: Why nested? + + Why create a ``polls`` directory under ``templates`` when we're + already inside the polls app? This directory is needed to avoid conflicts in + Django's ``app_directories`` template loader. For example, if two + apps had a template called ``base.html``, without the extra directory it + wouldn't be possible to distinguish between the two. It's a good convention + to use the name of your app for this directory. + +.. _installing-reusable-apps-prerequisites: + +Installing some prerequisites +============================= + +The current state of Python packaging is a bit muddled with various tools. For +this tutorial, we're going to use distribute_ to build our package. It's a +community-maintained fork of the older ``setuptools`` project. We'll also be +using `pip`_ to uninstall it after we're finished. You should install these +two packages now. If you need help, you can refer to :ref:`how to install +Django with pip`. You can install ``distribute`` +the same way. + +.. _distribute: http://pypi.python.org/pypi/distribute +.. _pip: http://pypi.python.org/pypi/pip + +Packaging your app +================== + +Python *packaging* refers to preparing your app in a specific format that can +be easily installed and used. Django itself is packaged very much like +this. For a small app like polls, this process isn't too difficult. + +1. First, create a parent directory for ``polls``, outside of your Django + project. Call this directory ``django-polls``. + +.. admonition:: Choosing a name for your app + + When choosing a name for your package, check resources like PyPI to avoid + naming conflicts with existing packages. It's often useful to prepend + ``django-`` to your module name when creating a package to distribute. + This helps others looking for Django apps identify your app as Django + specific. + +2. Move the ``polls`` directory into the ``django-polls`` directory. + +3. Create a file ``django-polls/README.txt`` with the following contents:: + + ===== + Polls + ===== + + Polls is a simple Django app to conduct Web-based polls. For each + question, visitors can choose between a fixed number of answers. + + Detailed documentation is in the "docs" directory. + + Quick start + ----------- + + 1. Add "polls" to your INSTALLED_APPS setting like this:: + + INSTALLED_APPS = ( + ... + 'polls', + ) + + 2. Include the polls URLconf in your project urls.py like this:: + + url(r'^polls/', include('polls.urls')), + + 3. Run `python manage.py syncdb` to create the polls models. + + 4. Start the development server and visit http://127.0.0.1:8000/admin/ + to create a poll (you'll need the Admin app enabled). + + 5. Visit http://127.0.0.1:8000/polls/ to participate in the poll. + +4. Create a ``django-polls/LICENSE`` file. Choosing a license is beyond the +scope of this tutorial, but suffice it to say that code released publicly +without a license is *useless*. Django and many Django-compatible apps are +distributed under the BSD license; however, you're free to pick your own +license. Just be aware that your licensing choice will affect who is able +to use your code. + +5. Next we'll create a ``setup.py`` file which provides details about how to +build and install the app. A full explanation of this file is beyond the +scope of this tutorial, but the `distribute docs +`_ have a good explanation. +Create a file ``django-polls/setup.py`` with the following contents:: + + import os + from setuptools import setup + + README = open(os.path.join(os.path.dirname(__file__), 'README.txt')).read() + + # allow setup.py to be run from any path + os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir))) + + setup( + name = 'django-polls', + version = '0.1', + packages = ['polls'], + include_package_data = True, + license = 'BSD License', # example license + description = 'A simple Django app to conduct Web-based polls.', + long_description = README, + url = 'http://www.example.com/', + author = 'Your Name', + author_email = 'yourname@example.com', + classifiers = [ + 'Environment :: Web Environment', + 'Framework :: Django', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', # example license + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + ], + ) + +.. admonition:: I thought you said we were going to use ``distribute``? + + Distribute is a drop-in replacement for ``setuptools``. Even though we + appear to import from ``setuptools``, since we have ``distribute`` + installed, it will override the import. + +6. Only Python modules and packages are included in the package by default. To + include additional files, we'll need to create a ``MANIFEST.in`` file. The + distribute docs referred to in the previous step discuss this file in more + details. To include the templates and our LICENSE file, create a file + ``django-polls/MANIFEST.in`` with the following contents:: + + include LICENSE + recursive-include polls/templates * + +7. It's optional, but recommended, to include detailed documentation with your + app. Create an empty directory ``django-polls/docs`` for future + documentation. Add an additional line to ``django-polls/MANIFEST.in``:: + + recursive-include docs * + + Note that the ``docs`` directory won't be included in your package unless + you add some files to it. Many Django apps also provide their documentation + online through sites like `readthedocs.org `_. + +8. Try building your package with ``python setup.py sdist`` (run from inside + ``django-polls``). This creates a directory called ``dist`` and builds your + new package, ``django-polls-0.1.tar.gz``. + +For more information on packaging, see `The Hitchhiker's Guide to Packaging +`_. + +Using your own package +====================== + +Since we moved the ``polls`` directory out of the project, it's no longer +working. We'll now fix this by installing our new ``django-polls`` package. + +.. admonition:: Installing as a system library + + The following steps install ``django-polls`` as a system library. In + general, it's best to avoid messing with your system libraries to avoid + breaking things. For this simple example though, the risk is low and it will + help with understanding packaging. We'll explain how to uninstall in + step 4. + + For experienced users, a neater way to manage your packages is to use + "virtualenv" (see below). + +1. Inside ``django-polls/dist``, untar the new package + ``django-polls-0.1.tar.gz`` (e.g. ``tar xzvf django-polls-0.1.tar.gz``). If + you're using Windows, you can download the command-line tool bsdtar_ to do + this, or you can use a GUI-based tool such as 7-zip_. + +2. Change into the directory created in step 1 (e.g. ``cd django-polls-0.1``). + +3. If you're using GNU/Linux, Mac OS X or some other flavor of Unix, enter the + command ``sudo python setup.py install`` at the shell prompt. If you're + using Windows, start up a command shell with administrator privileges and + run the command ``setup.py install``. + + With luck, your Django project should now work correctly again. Run the + server again to confirm this. + +4. To uninstall the package, use pip (you already :ref:`installed it + `, right?):: + + sudo pip uninstall django-polls + +.. _bsdtar: http://gnuwin32.sourceforge.net/packages/bsdtar.htm +.. _7-zip: http://www.7-zip.org/ +.. _pip: http://pypi.python.org/pypi/pip + +Publishing your app +=================== + +Now that we've packaged and tested ``django-polls``, it's ready to share with +the world! If this wasn't just an example, you could now: + +* Email the package to a friend. + +* Upload the package on your Web site. + +* Post the package on a public repository, such as `The Python Package Index + (PyPI) `_. + +For more information on PyPI, see the `Quickstart +`_ +section of The Hitchhiker's Guide to Packaging. One detail this guide mentions +is choosing the license under which your code is distributed. + +Installing Python packages with virtualenv +========================================== + +Earlier, we installed the polls app as a system library. This has some +disadvantages: + +* Modifying the system libraries can affect other Python software on your + system. + +* You won't be able to run multiple versions of this package (or others with + the same name). + +Typically, these situations only arise once you're maintaining several Django +projects. When they do, the best solution is to use `virtualenv +`_. This tool allows you to maintain multiple +isolated Python environments, each with its own copy of the libraries and +package namespace. diff --git a/docs/intro/tutorial03.txt b/docs/intro/tutorial03.txt index 169e6cd59f..5adfc9a490 100644 --- a/docs/intro/tutorial03.txt +++ b/docs/intro/tutorial03.txt @@ -315,6 +315,12 @@ Load the page in your Web browser, and you should see a bulleted-list containing the "What's up" poll from Tutorial 1. The link points to the poll's detail page. +.. admonition:: Organizing Templates + + Rather than one big templates directory, you can also store templates + within each app. We'll discuss this in more detail in the :doc:`reusable + apps tutorial`. + A shortcut: :func:`~django.shortcuts.render` -------------------------------------------- diff --git a/docs/intro/tutorial04.txt b/docs/intro/tutorial04.txt index 8909caf98b..dfee827056 100644 --- a/docs/intro/tutorial04.txt +++ b/docs/intro/tutorial04.txt @@ -278,5 +278,10 @@ For full details on generic views, see the :doc:`generic views documentation What's next? ============ -The tutorial ends here for the time being. In the meantime, you might want to -check out some pointers on :doc:`where to go from here `. +The beginner tutorial ends here for the time being. In the meantime, you might +want to check out some pointers on :doc:`where to go from here +`. + +If you are familiar with Python packaging and interested in learning how to +turn polls into a "reusable app", check out :doc:`Advanced tutorial: How to +write reusable apps`. From d55c54a5da241e294c1162a19a07ab3f59e58dc7 Mon Sep 17 00:00:00 2001 From: Brent O'Connor Date: Tue, 30 Oct 2012 16:19:31 -0700 Subject: [PATCH 049/302] The timeout variable wasn't defined, which was a little confusing. --- docs/topics/testing.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/testing.txt b/docs/topics/testing.txt index f5fd4fe3e6..8fccf32946 100644 --- a/docs/topics/testing.txt +++ b/docs/topics/testing.txt @@ -2084,6 +2084,7 @@ out the `full reference`_ for more details. def test_login(self): from selenium.webdriver.support.wait import WebDriverWait + timeout = 2 ... self.selenium.find_element_by_xpath('//input[@value="Log in"]').click() # Wait until the response is received From 68847135bc9acb2c51c2d36797d0a85395f0cd35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Fri, 10 Aug 2012 22:00:21 +0300 Subject: [PATCH 050/302] Removed dupe_avoidance from sql/query and sql/compiler.py The dupe avoidance logic was removed as it doesn't seem to do anything, it is complicated, and it has nearly zero documentation. The removal of dupe_avoidance allowed for refactoring of both the implementation and signature of Query.join(). This refactoring cascades again to some other parts. The most significant of them is the changes in qs.combine(), and compiler.select_related_descent(). --- django/db/models/fields/related.py | 11 -- django/db/models/options.py | 19 --- django/db/models/sql/compiler.py | 68 ++------- django/db/models/sql/constants.py | 3 + django/db/models/sql/query.py | 196 ++++++++++--------------- tests/regressiontests/queries/tests.py | 12 ++ 6 files changed, 98 insertions(+), 211 deletions(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 80c62f85c4..5c3f538018 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -1053,11 +1053,6 @@ class ForeignKey(RelatedField, Field): def contribute_to_class(self, cls, name): super(ForeignKey, self).contribute_to_class(cls, name) setattr(cls, self.name, ReverseSingleRelatedObjectDescriptor(self)) - if isinstance(self.rel.to, six.string_types): - target = self.rel.to - else: - target = self.rel.to._meta.db_table - cls._meta.duplicate_targets[self.column] = (target, "o2m") def contribute_to_related_class(self, cls, related): # Internal FK's - i.e., those with a related name ending with '+' - @@ -1293,12 +1288,6 @@ class ManyToManyField(RelatedField, Field): field.rel.through = model add_lazy_relation(cls, self, self.rel.through, resolve_through_model) - if isinstance(self.rel.to, six.string_types): - target = self.rel.to - else: - target = self.rel.to._meta.db_table - cls._meta.duplicate_targets[self.column] = (target, "m2m") - def contribute_to_related_class(self, cls, related): # Internal M2Ms (i.e., those with a related name ending with '+') # and swapped models don't get a related descriptor. diff --git a/django/db/models/options.py b/django/db/models/options.py index d471bba262..f430caceef 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -58,7 +58,6 @@ class Options(object): self.concrete_model = None self.swappable = None self.parents = SortedDict() - self.duplicate_targets = {} self.auto_created = False # To handle various inheritance situations, we need to track where @@ -147,24 +146,6 @@ class Options(object): auto_created=True) model.add_to_class('id', auto) - # Determine any sets of fields that are pointing to the same targets - # (e.g. two ForeignKeys to the same remote model). The query - # construction code needs to know this. At the end of this, - # self.duplicate_targets will map each duplicate field column to the - # columns it duplicates. - collections = {} - for column, target in six.iteritems(self.duplicate_targets): - try: - collections[target].add(column) - except KeyError: - collections[target] = set([column]) - self.duplicate_targets = {} - for elt in six.itervalues(collections): - if len(elt) == 1: - continue - for column in elt: - self.duplicate_targets[column] = elt.difference(set([column])) - def add_field(self, field): # Insert the given field in the order in which it was created, using # the "creation_counter" attribute of the field. diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 7461f5f31d..db1e1131ad 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -6,7 +6,7 @@ from django.db.backends.util import truncate_name from django.db.models.constants import LOOKUP_SEP from django.db.models.query_utils import select_related_descend from django.db.models.sql.constants import (SINGLE, MULTI, ORDER_DIR, - GET_ITERATOR_CHUNK_SIZE, SelectInfo) + GET_ITERATOR_CHUNK_SIZE, REUSE_ALL, SelectInfo) from django.db.models.sql.datastructures import EmptyResultSet from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.query import get_order_dir, Query @@ -457,7 +457,7 @@ class SQLCompiler(object): if not alias: alias = self.query.get_initial_alias() field, target, opts, joins, _, _ = self.query.setup_joins(pieces, - opts, alias, False) + opts, alias, REUSE_ALL) # We will later on need to promote those joins that were added to the # query afresh above. joins_to_promote = [j for j in joins if self.query.alias_refcount[j] < 2] @@ -574,8 +574,7 @@ class SQLCompiler(object): return result, params def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, - used=None, requested=None, restricted=None, nullable=None, - dupe_set=None, avoid_set=None): + requested=None, restricted=None, nullable=None): """ Fill in the information needed for a select_related query. The current depth is measured as the number of connections away from the root model @@ -590,13 +589,6 @@ class SQLCompiler(object): opts = self.query.get_meta() root_alias = self.query.get_initial_alias() self.query.related_select_cols = [] - if not used: - used = set() - if dupe_set is None: - dupe_set = set() - if avoid_set is None: - avoid_set = set() - orig_dupe_set = dupe_set only_load = self.query.get_loaded_field_names() # Setup for the case when only particular related fields should be @@ -616,12 +608,6 @@ class SQLCompiler(object): if not select_related_descend(f, restricted, requested, only_load.get(field_model)): continue - # The "avoid" set is aliases we want to avoid just for this - # particular branch of the recursion. They aren't permanently - # forbidden from reuse in the related selection tables (which is - # what "used" specifies). - avoid = avoid_set.copy() - dupe_set = orig_dupe_set.copy() table = f.rel.to._meta.db_table promote = nullable or f.null if model: @@ -637,31 +623,17 @@ class SQLCompiler(object): int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), lhs_col), - ())) - dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join((alias, int_opts.db_table, lhs_col, - int_opts.pk.column), exclusions=used, + int_opts.pk.column), promote=promote) alias_chain.append(alias) - for (dupe_opts, dupe_col) in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) else: alias = root_alias - dedupe = f.column in opts.duplicate_targets - if dupe_set or dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) - if dedupe: - dupe_set.add((opts, f.column)) - alias = self.query.join((alias, table, f.column, f.rel.get_related_field().column), - exclusions=used.union(avoid), promote=promote) - used.add(alias) + promote=promote) columns, aliases = self.get_default_columns(start_alias=alias, opts=f.rel.to._meta, as_pairs=True) self.query.related_select_cols.extend( @@ -671,10 +643,8 @@ class SQLCompiler(object): else: next = False new_nullable = f.null or promote - for dupe_opts, dupe_col in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, - used, next, restricted, new_nullable, dupe_set, avoid) + next, restricted, new_nullable) if restricted: related_fields = [ @@ -686,14 +656,8 @@ class SQLCompiler(object): if not select_related_descend(f, restricted, requested, only_load.get(model), reverse=True): continue - # The "avoid" set is aliases we want to avoid just for this - # particular branch of the recursion. They aren't permanently - # forbidden from reuse in the related selection tables (which is - # what "used" specifies). - avoid = avoid_set.copy() - dupe_set = orig_dupe_set.copy() - table = model._meta.db_table + table = model._meta.db_table int_opts = opts alias = root_alias alias_chain = [] @@ -708,30 +672,16 @@ class SQLCompiler(object): int_opts = int_model._meta continue lhs_col = int_opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - avoid.update((self.query.dupe_avoidance.get(id(opts), lhs_col), - ())) - dupe_set.add((opts, lhs_col)) int_opts = int_model._meta alias = self.query.join( (alias, int_opts.db_table, lhs_col, int_opts.pk.column), - exclusions=used, promote=True, reuse=used + promote=True, ) alias_chain.append(alias) - for dupe_opts, dupe_col in dupe_set: - self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) - dedupe = f.column in opts.duplicate_targets - if dupe_set or dedupe: - avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) - if dedupe: - dupe_set.add((opts, f.column)) alias = self.query.join( (alias, table, f.rel.get_related_field().column, f.column), - exclusions=used.union(avoid), promote=True ) - used.add(alias) columns, aliases = self.get_default_columns(start_alias=alias, opts=model._meta, as_pairs=True, local_only=True) self.query.related_select_cols.extend( @@ -743,7 +693,7 @@ class SQLCompiler(object): new_nullable = True self.fill_related_selections(model._meta, table, cur_depth+1, - used, next, restricted, new_nullable) + next, restricted, new_nullable) def deferred_to_columns(self): """ diff --git a/django/db/models/sql/constants.py b/django/db/models/sql/constants.py index 7e34047e1d..6e1d2dd87a 100644 --- a/django/db/models/sql/constants.py +++ b/django/db/models/sql/constants.py @@ -37,3 +37,6 @@ ORDER_DIR = { 'ASC': ('ASC', 'DESC'), 'DESC': ('DESC', 'ASC'), } + +# A marker for join-reusability. +REUSE_ALL = object() diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index de7e5904a3..ce45ec314a 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -20,7 +20,7 @@ from django.db.models.expressions import ExpressionNode from django.db.models.fields import FieldDoesNotExist from django.db.models.sql import aggregates as base_aggregates_module from django.db.models.sql.constants import (QUERY_TERMS, ORDER_DIR, SINGLE, - ORDER_PATTERN, JoinInfo, SelectInfo) + ORDER_PATTERN, REUSE_ALL, JoinInfo, SelectInfo) from django.db.models.sql.datastructures import EmptyResultSet, Empty, MultiJoin from django.db.models.sql.expressions import SQLEvaluator from django.db.models.sql.where import (WhereNode, Constraint, EverythingNode, @@ -115,7 +115,6 @@ class Query(object): self.default_ordering = True self.standard_ordering = True self.ordering_aliases = [] - self.dupe_avoidance = {} self.used_aliases = set() self.filter_is_sticky = False self.included_inherited_models = {} @@ -257,7 +256,6 @@ class Query(object): obj.standard_ordering = self.standard_ordering obj.included_inherited_models = self.included_inherited_models.copy() obj.ordering_aliases = [] - obj.dupe_avoidance = self.dupe_avoidance.copy() obj.select = self.select[:] obj.related_select_cols = [] obj.tables = self.tables[:] @@ -460,24 +458,42 @@ class Query(object): self.remove_inherited_models() # Work out how to relabel the rhs aliases, if necessary. change_map = {} - used = set() conjunction = (connector == AND) - # Add the joins in the rhs query into the new query. - first = True - for alias in rhs.tables: + + # Determine which existing joins can be reused. When combining the + # query with AND we must recreate all joins for m2m filters. When + # combining with OR we can reuse joins. The reason is that in AND + # case a single row can't fulfill a condition like: + # revrel__col=1 & revrel__col=2 + # But, there might be two different related rows matching this + # condition. In OR case a single True is enough, so single row is + # enough, too. + # + # Note that we will be creating duplicate joins for non-m2m joins in + # the AND case. The results will be correct but this creates too many + # joins. This is something that could be fixed later on. + reuse = set() if conjunction else set(self.tables) + # Base table must be present in the query - this is the same + # table on both sides. + self.get_initial_alias() + # Now, add the joins from rhs query into the new query (skipping base + # table). + for alias in rhs.tables[1:]: if not rhs.alias_refcount[alias]: - # An unused alias. continue - table, _, join_type, lhs, lhs_col, col, _ = rhs.alias_map[alias] - promote = join_type == self.LOUTER + table, _, join_type, lhs, lhs_col, col, nullable = rhs.alias_map[alias] + promote = (join_type == self.LOUTER) # If the left side of the join was already relabeled, use the # updated alias. lhs = change_map.get(lhs, lhs) - new_alias = self.join((lhs, table, lhs_col, col), - conjunction and not first, used, promote, not conjunction) - used.add(new_alias) + new_alias = self.join( + (lhs, table, lhs_col, col), reuse=reuse, promote=promote, + outer_if_first=not conjunction, nullable=nullable) + # We can't reuse the same join again in the query. If we have two + # distinct joins for the same connection in rhs query, then the + # combined query must have two joins, too. + reuse.discard(new_alias) change_map[alias] = new_alias - first = False # So that we don't exclude valid results in an "or" query combination, # all joins exclusive to either the lhs or the rhs must be converted @@ -767,9 +783,11 @@ class Query(object): (key, relabel_column(col)) for key, col in self.aggregates.items()) # 2. Rename the alias in the internal table/alias datastructures. - for k, aliases in self.join_map.items(): + for ident, aliases in self.join_map.items(): + del self.join_map[ident] aliases = tuple([change_map.get(a, a) for a in aliases]) - self.join_map[k] = aliases + ident = (change_map.get(ident[0], ident[0]),) + ident[1:] + self.join_map[ident] = aliases for old_alias, new_alias in six.iteritems(change_map): alias_data = self.alias_map[old_alias] alias_data = alias_data._replace(rhs_alias=new_alias) @@ -844,8 +862,8 @@ class Query(object): """ return len([1 for count in six.itervalues(self.alias_refcount) if count]) - def join(self, connection, always_create=False, exclusions=(), - promote=False, outer_if_first=False, nullable=False, reuse=None): + def join(self, connection, reuse=REUSE_ALL, promote=False, + outer_if_first=False, nullable=False): """ Returns an alias for the join in 'connection', either reusing an existing alias for that join or creating a new one. 'connection' is a @@ -855,56 +873,40 @@ class Query(object): lhs.lhs_col = table.col - If 'always_create' is True and 'reuse' is None, a new alias is always - created, regardless of whether one already exists or not. If - 'always_create' is True and 'reuse' is a set, an alias in 'reuse' that - matches the connection will be returned, if possible. If - 'always_create' is False, the first existing alias that matches the - 'connection' is returned, if any. Otherwise a new join is created. - - If 'exclusions' is specified, it is something satisfying the container - protocol ("foo in exclusions" must work) and specifies a list of - aliases that should not be returned, even if they satisfy the join. + The 'reuse' parameter can be used in three ways: it can be REUSE_ALL + which means all joins (matching the connection) are reusable, it can + be a set containing the aliases that can be reused, or it can be None + which means a new join is always created. If 'promote' is True, the join type for the alias will be LOUTER (if the alias previously existed, the join type will be promoted from INNER to LOUTER, if necessary). If 'outer_if_first' is True and a new join is created, it will have the - LOUTER join type. This is used when joining certain types of querysets - and Q-objects together. + LOUTER join type. Used for example when adding ORed filters, where we + want to use LOUTER joins except if some other join already restricts + the join to INNER join. A join is always created as LOUTER if the lhs alias is LOUTER to make - sure we do not generate chains like a LOUTER b INNER c. + sure we do not generate chains like t1 LOUTER t2 INNER t3. If 'nullable' is True, the join can potentially involve NULL values and is a candidate for promotion (to "left outer") when combining querysets. """ lhs, table, lhs_col, col = connection - if lhs in self.alias_map: - lhs_table = self.alias_map[lhs].table_name + existing = self.join_map.get(connection, ()) + if reuse == REUSE_ALL: + reuse = existing + elif reuse is None: + reuse = set() else: - lhs_table = lhs - - if reuse and always_create and table in self.table_map: - # Convert the 'reuse' to case to be "exclude everything but the - # reusable set, minus exclusions, for this table". - exclusions = set(self.table_map[table]).difference(reuse).union(set(exclusions)) - always_create = False - t_ident = (lhs_table, table, lhs_col, col) - if not always_create: - for alias in self.join_map.get(t_ident, ()): - if alias not in exclusions: - if lhs_table and not self.alias_refcount[self.alias_map[alias].lhs_alias]: - # The LHS of this join tuple is no longer part of the - # query, so skip this possibility. - continue - if self.alias_map[alias].lhs_alias != lhs: - continue - self.ref_alias(alias) - if promote or (lhs and self.alias_map[lhs].join_type == self.LOUTER): - self.promote_joins([alias]) - return alias + reuse = [a for a in existing if a in reuse] + if reuse: + alias = reuse[0] + self.ref_alias(alias) + if promote or (lhs and self.alias_map[lhs].join_type == self.LOUTER): + self.promote_joins([alias]) + return alias # No reuse is possible, so we need a new alias. alias, _ = self.table_alias(table, True) @@ -915,18 +917,16 @@ class Query(object): elif (promote or outer_if_first or self.alias_map[lhs].join_type == self.LOUTER): # We need to use LOUTER join if asked by promote or outer_if_first, - # or if the LHS table is left-joined in the query. Adding inner join - # to an existing outer join effectively cancels the effect of the - # outer join. + # or if the LHS table is left-joined in the query. join_type = self.LOUTER else: join_type = self.INNER join = JoinInfo(table, alias, join_type, lhs, lhs_col, col, nullable) self.alias_map[alias] = join - if t_ident in self.join_map: - self.join_map[t_ident] += (alias,) + if connection in self.join_map: + self.join_map[connection] += (alias,) else: - self.join_map[t_ident] = (alias,) + self.join_map[connection] = (alias,) return alias def setup_inherited_models(self): @@ -1003,7 +1003,7 @@ class Query(object): # then we need to explore the joins that are required. field, source, opts, join_list, last, _ = self.setup_joins( - field_list, opts, self.get_initial_alias(), False) + field_list, opts, self.get_initial_alias(), REUSE_ALL) # Process the join chain to see if it can be trimmed col, _, join_list = self.trim_joins(source, join_list, last, False) @@ -1114,8 +1114,8 @@ class Query(object): try: field, target, opts, join_list, last, extra_filters = self.setup_joins( - parts, opts, alias, True, allow_many, allow_explicit_fk=True, - can_reuse=can_reuse, negate=negate, + parts, opts, alias, can_reuse, allow_many, + allow_explicit_fk=True, negate=negate, process_extras=process_extras) except MultiJoin as e: self.split_exclude(filter_expr, LOOKUP_SEP.join(parts[:e.level]), @@ -1268,9 +1268,8 @@ class Query(object): if self.filter_is_sticky: self.used_aliases = used_aliases - def setup_joins(self, names, opts, alias, dupe_multis, allow_many=True, - allow_explicit_fk=False, can_reuse=None, negate=False, - process_extras=True): + def setup_joins(self, names, opts, alias, can_reuse, allow_many=True, + allow_explicit_fk=False, negate=False, process_extras=True): """ Compute the necessary table joins for the passage through the fields given in 'names'. 'opts' is the Options class for the current model @@ -1290,14 +1289,9 @@ class Query(object): """ joins = [alias] last = [0] - dupe_set = set() - exclusions = set() extra_filters = [] int_alias = None for pos, name in enumerate(names): - if int_alias is not None: - exclusions.add(int_alias) - exclusions.add(alias) last.append(len(joins)) if name == 'pk': name = opts.pk.name @@ -1330,28 +1324,12 @@ class Query(object): opts = int_model._meta else: lhs_col = opts.parents[int_model].column - dedupe = lhs_col in opts.duplicate_targets - if dedupe: - exclusions.update(self.dupe_avoidance.get( - (id(opts), lhs_col), ())) - dupe_set.add((opts, lhs_col)) opts = int_model._meta alias = self.join((alias, opts.db_table, lhs_col, - opts.pk.column), exclusions=exclusions) + opts.pk.column)) joins.append(alias) - exclusions.add(alias) - for (dupe_opts, dupe_col) in dupe_set: - self.update_dupe_avoidance(dupe_opts, dupe_col, - alias) cached_data = opts._join_cache.get(name) orig_opts = opts - dupe_col = direct and field.column or field.field.column - dedupe = dupe_col in opts.duplicate_targets - if dupe_set or dedupe: - if dedupe: - dupe_set.add((opts, dupe_col)) - exclusions.update(self.dupe_avoidance.get((id(opts), dupe_col), - ())) if process_extras and hasattr(field, 'extra_filters'): extra_filters.extend(field.extra_filters(names, pos, negate)) @@ -1377,16 +1355,14 @@ class Query(object): target) int_alias = self.join((alias, table1, from_col1, to_col1), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) + reuse=can_reuse, nullable=True) if int_alias == table2 and from_col2 == to_col2: joins.append(int_alias) alias = int_alias else: alias = self.join( (int_alias, table2, from_col2, to_col2), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) + reuse=can_reuse, nullable=True) joins.extend([int_alias, alias]) elif field.rel: # One-to-one or many-to-one field @@ -1402,7 +1378,6 @@ class Query(object): opts, target) alias = self.join((alias, table, from_col, to_col), - exclusions=exclusions, nullable=self.is_nullable(field)) joins.append(alias) else: @@ -1433,11 +1408,9 @@ class Query(object): target) int_alias = self.join((alias, table1, from_col1, to_col1), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) + reuse=can_reuse, nullable=True) alias = self.join((int_alias, table2, from_col2, to_col2), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) + reuse=can_reuse, nullable=True) joins.extend([int_alias, alias]) else: # One-to-many field (ForeignKey defined on the target model) @@ -1461,17 +1434,9 @@ class Query(object): opts, target) alias = self.join((alias, table, from_col, to_col), - dupe_multis, exclusions, nullable=True, - reuse=can_reuse) + reuse=can_reuse, nullable=True) joins.append(alias) - for (dupe_opts, dupe_col) in dupe_set: - if int_alias is None: - to_avoid = alias - else: - to_avoid = int_alias - self.update_dupe_avoidance(dupe_opts, dupe_col, to_avoid) - if pos != len(names) - 1: if pos == len(names) - 2: raise FieldError("Join on field %r not permitted. Did you misspell %r for the lookup type?" % (name, names[pos + 1])) @@ -1538,19 +1503,6 @@ class Query(object): penultimate = last.pop() return col, alias, join_list - def update_dupe_avoidance(self, opts, col, alias): - """ - For a column that is one of multiple pointing to the same table, update - the internal data structures to note that this alias shouldn't be used - for those other columns. - """ - ident = id(opts) - for name in opts.duplicate_targets[col]: - try: - self.dupe_avoidance[ident, name].add(alias) - except KeyError: - self.dupe_avoidance[ident, name] = set([alias]) - def split_exclude(self, filter_expr, prefix, can_reuse): """ When doing an exclude against any kind of N-to-many relation, we need @@ -1657,8 +1609,8 @@ class Query(object): try: for name in field_names: field, target, u2, joins, u3, u4 = self.setup_joins( - name.split(LOOKUP_SEP), opts, alias, False, allow_m2m, - True) + name.split(LOOKUP_SEP), opts, alias, REUSE_ALL, + allow_m2m, True) final_alias = joins[-1] col = target.column if len(joins) > 1: @@ -1948,7 +1900,7 @@ class Query(object): opts = self.model._meta alias = self.get_initial_alias() field, col, opts, joins, last, extra = self.setup_joins( - start.split(LOOKUP_SEP), opts, alias, False) + start.split(LOOKUP_SEP), opts, alias, REUSE_ALL) select_col = self.alias_map[joins[1]].lhs_join_col select_alias = alias diff --git a/tests/regressiontests/queries/tests.py b/tests/regressiontests/queries/tests.py index c2a4ad1caf..2dbe75fd24 100644 --- a/tests/regressiontests/queries/tests.py +++ b/tests/regressiontests/queries/tests.py @@ -1046,6 +1046,18 @@ class Queries4Tests(BaseQuerysetTest): self.assertQuerysetEqual(q1, [""]) self.assertEqual(str(q1.query), str(q2.query)) + def test_combine_join_reuse(self): + # Test that we correctly recreate joins having identical connections + # in the rhs query, in case the query is ORed together. Related to + # ticket #18748 + Report.objects.create(name='r4', creator=self.a1) + q1 = Author.objects.filter(report__name='r5') + q2 = Author.objects.filter(report__name='r4').filter(report__name='r1') + combined = q1|q2 + self.assertEquals(str(combined.query).count('JOIN'), 2) + self.assertEquals(len(combined), 1) + self.assertEquals(combined[0].name, 'a1') + def test_ticket7095(self): # Updates that are filtered on the model being updated are somewhat # tricky in MySQL. This exercises that case. From 146ed13a111c97c1c04902a6c0eda1e4ee6e604c Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 30 Oct 2012 21:59:23 +0100 Subject: [PATCH 051/302] Fixed #17083 -- Allowed sessions to use non-default cache. --- django/conf/global_settings.py | 1 + django/contrib/sessions/backends/cache.py | 5 +++-- django/contrib/sessions/tests.py | 26 +++++++++++++++++++---- docs/ref/settings.txt | 10 +++++++++ docs/releases/1.5.txt | 3 +++ docs/topics/http/sessions.txt | 9 ++++++++ 6 files changed, 48 insertions(+), 6 deletions(-) diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 84296c7493..c533efc41c 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -445,6 +445,7 @@ MIDDLEWARE_CLASSES = ( # SESSIONS # ############ +SESSION_CACHE_ALIAS = 'default' # Cache to store session data if using the cache session backend. SESSION_COOKIE_NAME = 'sessionid' # Cookie name. This can be whatever you want. SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 2 # Age of cookie, in seconds (default: 2 weeks). SESSION_COOKIE_DOMAIN = None # A string like ".example.com", or None for standard domain cookie. diff --git a/django/contrib/sessions/backends/cache.py b/django/contrib/sessions/backends/cache.py index 1b4906f923..596042fcb3 100644 --- a/django/contrib/sessions/backends/cache.py +++ b/django/contrib/sessions/backends/cache.py @@ -1,5 +1,6 @@ +from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError -from django.core.cache import cache +from django.core.cache import get_cache from django.utils.six.moves import xrange KEY_PREFIX = "django.contrib.sessions.cache" @@ -10,7 +11,7 @@ class SessionStore(SessionBase): A cache-based session store. """ def __init__(self, session_key=None): - self._cache = cache + self._cache = get_cache(settings.SESSION_CACHE_ALIAS) super(SessionStore, self).__init__(session_key) @property diff --git a/django/contrib/sessions/tests.py b/django/contrib/sessions/tests.py index 718967791d..da79ac9de6 100644 --- a/django/contrib/sessions/tests.py +++ b/django/contrib/sessions/tests.py @@ -13,8 +13,8 @@ from django.contrib.sessions.backends.file import SessionStore as FileSession from django.contrib.sessions.backends.signed_cookies import SessionStore as CookieSession from django.contrib.sessions.models import Session from django.contrib.sessions.middleware import SessionMiddleware +from django.core.cache import get_cache from django.core import management -from django.core.cache import DEFAULT_CACHE_ALIAS from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation from django.http import HttpResponse from django.test import TestCase, RequestFactory @@ -136,8 +136,8 @@ class SessionTestsMixin(object): self.assertTrue(self.session.modified) def test_save(self): - if (hasattr(self.session, '_cache') and - 'DummyCache' in settings.CACHES[DEFAULT_CACHE_ALIAS]['BACKEND']): + if (hasattr(self.session, '_cache') and'DummyCache' in + settings.CACHES[settings.SESSION_CACHE_ALIAS]['BACKEND']): raise unittest.SkipTest("Session saving tests require a real cache backend") self.session.save() self.assertTrue(self.session.exists(self.session.session_key)) @@ -355,7 +355,8 @@ class CacheDBSessionTests(SessionTestsMixin, TestCase): backend = CacheDBSession - @unittest.skipIf('DummyCache' in settings.CACHES[DEFAULT_CACHE_ALIAS]['BACKEND'], + @unittest.skipIf('DummyCache' in + settings.CACHES[settings.SESSION_CACHE_ALIAS]['BACKEND'], "Session saving tests require a real cache backend") def test_exists_searches_cache_first(self): self.session.save() @@ -454,6 +455,23 @@ class CacheSessionTests(SessionTestsMixin, unittest.TestCase): self.session._session_key = (string.ascii_letters + string.digits) * 20 self.assertEqual(self.session.load(), {}) + def test_default_cache(self): + self.session.save() + self.assertNotEqual(get_cache('default').get(self.session.cache_key), None) + + @override_settings(CACHES={ + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + }, + 'sessions': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + }, SESSION_CACHE_ALIAS='sessions') + def test_non_default_cache(self): + self.session.save() + self.assertEqual(get_cache('default').get(self.session.cache_key), None) + self.assertNotEqual(get_cache('sessions').get(self.session.cache_key), None) + class SessionMiddlewareTests(unittest.TestCase): diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index a909c12665..e8b41afb39 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1693,6 +1693,16 @@ This is useful if you have multiple Django instances running under the same hostname. They can use different cookie paths, and each instance will only see its own session cookie. +.. setting:: SESSION_CACHE_ALIAS + +SESSION_CACHE_ALIAS +------------------- + +Default: ``default`` + +If you're using :ref:`cache-based session storage `, +this selects the cache to use. + .. setting:: SESSION_COOKIE_SECURE SESSION_COOKIE_SECURE diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index 3ee1b2d21f..e18a78afc8 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -299,6 +299,9 @@ Django 1.5 also includes several smaller improvements worth noting: * RemoteUserMiddleware now forces logout when the REMOTE_USER header disappears during the same browser session. +* The :ref:`cache-based session backend ` can store + session data in a non-default cache. + Backwards incompatible changes in 1.5 ===================================== diff --git a/docs/topics/http/sessions.txt b/docs/topics/http/sessions.txt index d9c472d092..baf8aa5cb5 100644 --- a/docs/topics/http/sessions.txt +++ b/docs/topics/http/sessions.txt @@ -45,6 +45,8 @@ If you want to use a database-backed session, you need to add Once you have configured your installation, run ``manage.py syncdb`` to install the single database table that stores session data. +.. _cached-sessions-backend: + Using cached sessions --------------------- @@ -62,6 +64,13 @@ sure you've configured your cache; see the :doc:`cache documentation sessions directly instead of sending everything through the file or database cache backends. +If you have multiple caches defined in :setting:`CACHES`, Django will use the +default cache. To use another cache, set :setting:`SESSION_CACHE_ALIAS` to the +name of that cache. + +.. versionchanged:: 1.5 + The :setting:`SESSION_CACHE_ALIAS` setting was added. + Once your cache is configured, you've got two choices for how to store data in the cache: From 7f75460fd6befbef805fee3c91608efb0e9f444d Mon Sep 17 00:00:00 2001 From: Andrew Godwin Date: Wed, 31 Oct 2012 10:58:14 +0000 Subject: [PATCH 052/302] Fixed #19070 -- urlize filter no longer raises exceptions on 2.7 Thanks to claudep for the patch. --- django/utils/html.py | 2 +- tests/regressiontests/defaultfilters/tests.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/django/utils/html.py b/django/utils/html.py index cc8372906b..9816b9accb 100644 --- a/django/utils/html.py +++ b/django/utils/html.py @@ -18,7 +18,7 @@ from django.utils.text import normalize_newlines # Configuration for urlize() function. TRAILING_PUNCTUATION = ['.', ',', ':', ';', '.)'] -WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('<', '>')] +WRAPPING_PUNCTUATION = [('(', ')'), ('<', '>'), ('[', ']'), ('<', '>')] # List of possible strings used for bullets in bulleted lists. DOTS = ['·', '*', '\u2022', '•', '•', '•'] diff --git a/tests/regressiontests/defaultfilters/tests.py b/tests/regressiontests/defaultfilters/tests.py index d00203e304..52268da2ec 100644 --- a/tests/regressiontests/defaultfilters/tests.py +++ b/tests/regressiontests/defaultfilters/tests.py @@ -304,7 +304,12 @@ class DefaultFiltersTests(TestCase): # Check urlize trims trailing period when followed by parenthesis - see #18644 self.assertEqual(urlize('(Go to http://www.example.com/foo.)'), - '(Go to http://www.example.com/foo.)') + '(Go to http://www.example.com/foo.)') + + # Check urlize doesn't crash when square bracket is appended to url (#19070) + self.assertEqual(urlize('[see www.example.com]'), + '[see www.example.com]' ) + def test_wordcount(self): self.assertEqual(wordcount(''), 0) From ba81164fb771391b92575b5aaab6e286f56eb831 Mon Sep 17 00:00:00 2001 From: Kent Hauser Date: Wed, 24 Oct 2012 16:02:30 -0400 Subject: [PATCH 053/302] Add `form` to formwizard context (includes tests) --- django/contrib/formtools/tests/wizard/wizardtests/tests.py | 4 ++++ django/contrib/formtools/wizard/views.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/django/contrib/formtools/tests/wizard/wizardtests/tests.py b/django/contrib/formtools/tests/wizard/wizardtests/tests.py index 586bd59341..6403a5548d 100644 --- a/django/contrib/formtools/tests/wizard/wizardtests/tests.py +++ b/django/contrib/formtools/tests/wizard/wizardtests/tests.py @@ -72,6 +72,10 @@ class WizardTests(object): self.assertEqual(response.context['wizard']['steps'].current, 'form2') self.assertEqual(response.context.get('another_var', None), True) + # ticket #19025: `form` should be included in context + form = response.context_data['wizard']['form'] + self.assertEqual(response.context_data['form'], form) + def test_form_finish(self): response = self.client.get(self.wizard_url) self.assertEqual(response.status_code, 200) diff --git a/django/contrib/formtools/wizard/views.py b/django/contrib/formtools/wizard/views.py index ea41e86852..384366e251 100644 --- a/django/contrib/formtools/wizard/views.py +++ b/django/contrib/formtools/wizard/views.py @@ -528,7 +528,7 @@ class WizardView(TemplateView): context.update({'another_var': True}) return context """ - context = super(WizardView, self).get_context_data(**kwargs) + context = super(WizardView, self).get_context_data(form=form, **kwargs) context.update(self.storage.extra_data) context['wizard'] = { 'form': form, From c4b71beb658981011a455753bbb58024bfbeb713 Mon Sep 17 00:00:00 2001 From: Brett Koonce Date: Wed, 31 Oct 2012 15:40:08 -0500 Subject: [PATCH 054/302] minor fix (+'.' to end of line) --- docs/faq/general.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq/general.txt b/docs/faq/general.txt index 659a8c5ad9..dc569840d1 100644 --- a/docs/faq/general.txt +++ b/docs/faq/general.txt @@ -68,7 +68,7 @@ Who's behind this? Django was originally developed at World Online, the Web department of a newspaper in Lawrence, Kansas, USA. Django's now run by an international team of volunteers; you can read all about them over at the :doc:`list of committers -` +`. Which sites use Django? ----------------------- From dd0d2c0be56a28f868d501b06975984c138f4830 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 31 Oct 2012 19:56:53 -0400 Subject: [PATCH 055/302] Fixed #19216 - Switched to user level installation in apps tutorial. Thanks Nick Coghlan for the suggestion. --- docs/intro/reusable-apps.txt | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/intro/reusable-apps.txt b/docs/intro/reusable-apps.txt index 200051593c..11a441d277 100644 --- a/docs/intro/reusable-apps.txt +++ b/docs/intro/reusable-apps.txt @@ -291,16 +291,19 @@ Using your own package Since we moved the ``polls`` directory out of the project, it's no longer working. We'll now fix this by installing our new ``django-polls`` package. -.. admonition:: Installing as a system library +.. admonition:: Installing as a user library - The following steps install ``django-polls`` as a system library. In - general, it's best to avoid messing with your system libraries to avoid - breaking things. For this simple example though, the risk is low and it will - help with understanding packaging. We'll explain how to uninstall in - step 4. + The following steps install ``django-polls`` as a user library. Per-user + installs have a lot of advantages over installing the package system-wide, + such as being usable on systems where you don't have administrator access + as well as preventing the package from affecting system services and other + users of the machine. Python 2.6 added support for user libraries, so if + you are using an older version this won't work, but Django 1.5 requires + Python 2.6 or newer anyway. - For experienced users, a neater way to manage your packages is to use - "virtualenv" (see below). + Note that per-user installations can still affect the behavior of system + tools that run as that user, so ``virtualenv`` is a more robust solution + (see below). 1. Inside ``django-polls/dist``, untar the new package ``django-polls-0.1.tar.gz`` (e.g. ``tar xzvf django-polls-0.1.tar.gz``). If @@ -310,9 +313,9 @@ working. We'll now fix this by installing our new ``django-polls`` package. 2. Change into the directory created in step 1 (e.g. ``cd django-polls-0.1``). 3. If you're using GNU/Linux, Mac OS X or some other flavor of Unix, enter the - command ``sudo python setup.py install`` at the shell prompt. If you're - using Windows, start up a command shell with administrator privileges and - run the command ``setup.py install``. + command ``python setup.py install --user`` at the shell prompt. If you're + using Windows, start up a command shell and run the command + ``setup.py install --user``. With luck, your Django project should now work correctly again. Run the server again to confirm this. @@ -320,7 +323,7 @@ working. We'll now fix this by installing our new ``django-polls`` package. 4. To uninstall the package, use pip (you already :ref:`installed it `, right?):: - sudo pip uninstall django-polls + pip uninstall django-polls .. _bsdtar: http://gnuwin32.sourceforge.net/packages/bsdtar.htm .. _7-zip: http://www.7-zip.org/ @@ -347,11 +350,10 @@ is choosing the license under which your code is distributed. Installing Python packages with virtualenv ========================================== -Earlier, we installed the polls app as a system library. This has some +Earlier, we installed the polls app as a user library. This has some disadvantages: -* Modifying the system libraries can affect other Python software on your - system. +* Modifying the user libraries can affect other Python software on your system. * You won't be able to run multiple versions of this package (or others with the same name). From ede8a0be05f7b55d07ab5f60f3e8e3135a54f743 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Thu, 1 Nov 2012 06:58:02 -0400 Subject: [PATCH 056/302] Fixed #19179 - Added mention of NamedUrlSessionWizard and NamedUrlCookieWizard; thanks Tom for the report. --- docs/ref/contrib/formtools/form-wizard.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/ref/contrib/formtools/form-wizard.txt b/docs/ref/contrib/formtools/form-wizard.txt index 0ced1bf155..d1193badbe 100644 --- a/docs/ref/contrib/formtools/form-wizard.txt +++ b/docs/ref/contrib/formtools/form-wizard.txt @@ -622,8 +622,11 @@ Usage of ``NamedUrlWizardView`` .. class:: NamedUrlWizardView -There is a :class:`WizardView` subclass which adds named-urls support to the wizard. -By doing this, you can have single urls for every step. +There is a :class:`WizardView` subclass which adds named-urls support to the +wizard. By doing this, you can have single urls for every step. You can also +use the :class:`NamedUrlSessionWizardView` or :class:`NamedUrlCookieWizardView` +classes which preselect the backend used for storing information (server-side +sessions and browser cookies respectively). To use the named urls, you have to change the ``urls.py``. From d9213d09dbb28f687b53d051cddcae03337066c8 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 1 Nov 2012 19:44:45 +0100 Subject: [PATCH 057/302] Fixed #16678 -- Wrote tests for contrib.redirects app Thanks Julien Phalip for the report. --- django/contrib/redirects/tests.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 django/contrib/redirects/tests.py diff --git a/django/contrib/redirects/tests.py b/django/contrib/redirects/tests.py new file mode 100644 index 0000000000..11ffc7b748 --- /dev/null +++ b/django/contrib/redirects/tests.py @@ -0,0 +1,42 @@ +from django.conf import settings +from django.contrib.sites.models import Site +from django.test import TestCase +from django.test.utils import override_settings +from django.utils import six + +from .models import Redirect + + +@override_settings( + SITE_ID=1, + APPEND_SLASH=True, + MIDDLEWARE_CLASSES=list(settings.MIDDLEWARE_CLASSES) + + ['django.contrib.redirects.middleware.RedirectFallbackMiddleware'], +) +class RedirectTests(TestCase): + + def setUp(self): + self.site = Site.objects.get(pk=settings.SITE_ID) + + def test_model(self): + r1 = Redirect.objects.create( + site=self.site, old_path='/initial', new_path='/new_target') + self.assertEqual(six.text_type(r1), "/initial ---> /new_target") + + def test_redirect_middleware(self): + r1 = Redirect.objects.create( + site=self.site, old_path='/initial', new_path='/new_target') + response = self.client.get('/initial') + self.assertRedirects(response, + '/new_target', status_code=301, target_status_code=404) + # Works also with trailing slash + response = self.client.get('/initial/') + self.assertRedirects(response, + '/new_target', status_code=301, target_status_code=404) + + def test_response_gone(self): + """When the redirect target is '', return a 410""" + r1 = Redirect.objects.create( + site=self.site, old_path='/initial', new_path='') + response = self.client.get('/initial') + self.assertEqual(response.status_code, 410) From af7ea808d8540d1be87d89172f371ac928ff5c19 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Thu, 1 Nov 2012 16:11:05 -0400 Subject: [PATCH 058/302] Added WizardView.file_storage exception message and docs Thanks Danilo Bargen for the patch. --- .../contrib/formtools/wizard/storage/base.py | 8 ++++-- django/contrib/formtools/wizard/views.py | 12 +++++---- docs/ref/contrib/formtools/form-wizard.txt | 25 +++++++++++++++++++ docs/topics/files.txt | 2 ++ 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/django/contrib/formtools/wizard/storage/base.py b/django/contrib/formtools/wizard/storage/base.py index aafc833484..2e59679d09 100644 --- a/django/contrib/formtools/wizard/storage/base.py +++ b/django/contrib/formtools/wizard/storage/base.py @@ -69,7 +69,9 @@ class BaseStorage(object): wizard_files = self.data[self.step_files_key].get(step, {}) if wizard_files and not self.file_storage: - raise NoFileStorageConfigured + raise NoFileStorageConfigured( + "You need to define 'file_storage' in your " + "wizard view in order to handle file uploads.") files = {} for field, field_dict in six.iteritems(wizard_files): @@ -81,7 +83,9 @@ class BaseStorage(object): def set_step_files(self, step, files): if files and not self.file_storage: - raise NoFileStorageConfigured + raise NoFileStorageConfigured( + "You need to define 'file_storage' in your " + "wizard view in order to handle file uploads.") if step not in self.data[self.step_files_key]: self.data[self.step_files_key][step] = {} diff --git a/django/contrib/formtools/wizard/views.py b/django/contrib/formtools/wizard/views.py index ea41e86852..5b45267f36 100644 --- a/django/contrib/formtools/wizard/views.py +++ b/django/contrib/formtools/wizard/views.py @@ -174,7 +174,9 @@ class WizardView(TemplateView): for field in six.itervalues(form.base_fields): if (isinstance(field, forms.FileField) and not hasattr(cls, 'file_storage')): - raise NoFileStorageConfigured + raise NoFileStorageConfigured( + "You need to define 'file_storage' in your " + "wizard view in order to handle file uploads.") # build the kwargs for the wizardview instances kwargs['form_list'] = init_form_list @@ -436,8 +438,8 @@ class WizardView(TemplateView): def get_all_cleaned_data(self): """ Returns a merged dictionary of all step cleaned_data dictionaries. - If a step contains a `FormSet`, the key will be prefixed with formset - and contain a list of the formset cleaned_data dictionaries. + If a step contains a `FormSet`, the key will be prefixed with + 'formset-' and contain a list of the formset cleaned_data dictionaries. """ cleaned_data = {} for form_key in self.get_form_list(): @@ -458,8 +460,8 @@ class WizardView(TemplateView): def get_cleaned_data_for_step(self, step): """ Returns the cleaned data for a given `step`. Before returning the - cleaned data, the stored values are being revalidated through the - form. If the data doesn't validate, None will be returned. + cleaned data, the stored values are revalidated through the form. + If the data doesn't validate, None will be returned. """ if step in self.form_list: form_obj = self.get_form(step=step, diff --git a/docs/ref/contrib/formtools/form-wizard.txt b/docs/ref/contrib/formtools/form-wizard.txt index d1193badbe..3edc019d05 100644 --- a/docs/ref/contrib/formtools/form-wizard.txt +++ b/docs/ref/contrib/formtools/form-wizard.txt @@ -493,6 +493,21 @@ Advanced ``WizardView`` methods context = self.get_context_data(form=form, **kwargs) return self.render_to_response(context) +.. method:: WizardView.get_cleaned_data_for_step(step) + + This method returns the cleaned data for a given ``step``. Before returning + the cleaned data, the stored values are revalidated through the form. If + the data doesn't validate, ``None`` will be returned. + +.. method:: WizardView.get_all_cleaned_data() + + This method returns a merged dictionary of all form steps' ``cleaned_data`` + dictionaries. If a step contains a ``FormSet``, the key will be prefixed + with ``formset-`` and contain a list of the formset's ``cleaned_data`` + dictionaries. Note that if two or more steps have a field with the same + name, the value for that field from the latest step will overwrite the + value from any earlier steps. + Providing initial data for the forms ==================================== @@ -534,6 +549,16 @@ This storage will temporarily store the uploaded files for the wizard. The :attr:`file_storage` attribute should be a :class:`~django.core.files.storage.Storage` subclass. +Django provides a built-in storage class (see :ref:`the built-in filesystem +storage class `):: + + from django.conf import settings + from django.core.files.storage import FileSystemStorage + + class CustomWizardView(WizardView): + ... + file_storage = FileSystemStorage(location=os.path.join(settings.MEDIA_ROOT, 'photos')) + .. warning:: Please remember to take care of removing old files as the diff --git a/docs/topics/files.txt b/docs/topics/files.txt index c9b4327941..66e104759a 100644 --- a/docs/topics/files.txt +++ b/docs/topics/files.txt @@ -139,6 +139,8 @@ useful -- you can use the global default storage system:: See :doc:`/ref/files/storage` for the file storage API. +.. _builtin-fs-storage: + The built-in filesystem storage class ------------------------------------- From f975c4857d4442bf3d7c4cf1c28d317d26a2e297 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 2 Nov 2012 09:29:55 +0100 Subject: [PATCH 059/302] Fixed #19225 -- Typo in shortcuts docs. Thanks SunPowered for the report. --- docs/topics/http/shortcuts.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/http/shortcuts.txt b/docs/topics/http/shortcuts.txt index 0dc38b1459..b1b4700b73 100644 --- a/docs/topics/http/shortcuts.txt +++ b/docs/topics/http/shortcuts.txt @@ -4,7 +4,7 @@ Django shortcut functions .. module:: django.shortcuts :synopsis: - Convenience shortcuts that spam multiple levels of Django's MVC stack. + Convenience shortcuts that span multiple levels of Django's MVC stack. .. index:: shortcuts From 0d8432da552b2ddf2d2326edccae627dd05a414e Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Fri, 2 Nov 2012 10:51:10 +0100 Subject: [PATCH 060/302] Documented minimal python 3.2 version. --- docs/faq/install.txt | 8 ++++---- docs/topics/install.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/faq/install.txt b/docs/faq/install.txt index a772a379d5..a92c9d87ea 100644 --- a/docs/faq/install.txt +++ b/docs/faq/install.txt @@ -18,7 +18,7 @@ What are Django's prerequisites? Django requires Python, specifically Python 2.6.5 - 2.7.x. No other Python libraries are required for basic Django usage. Django 1.5 also has -experimental support for Python 3.2 and above. +experimental support for Python 3.2.3 and above. For a development environment -- if you just want to experiment with Django -- you don't need to have a separate Web server installed; Django comes with its @@ -69,14 +69,14 @@ Django version Python versions 1.2 2.4, 2.5, 2.6, 2.7 1.3 2.4, 2.5, 2.6, 2.7 **1.4** **2.5, 2.6, 2.7** -*1.5 (future)* *2.6, 2.7* and *3.2, 3.3 (experimental)* +*1.5 (future)* *2.6, 2.7* and *3.2.3, 3.3 (experimental)* ============== =============== Can I use Django with Python 3? ------------------------------- -Django 1.5 introduces experimental support for Python 3.2 and 3.3. However, we -don't yet suggest that you use Django and Python 3 in production. +Django 1.5 introduces experimental support for Python 3.2.3 and above. However, +we don't yet suggest that you use Django and Python 3 in production. Python 3 support should be considered a "preview". It's offered to bootstrap the transition of the Django ecosystem to Python 3, and to help you start diff --git a/docs/topics/install.txt b/docs/topics/install.txt index 52994ed16a..976e29beeb 100644 --- a/docs/topics/install.txt +++ b/docs/topics/install.txt @@ -10,7 +10,7 @@ Install Python Being a Python Web framework, Django requires Python. It works with any Python version from 2.6.5 to 2.7. It also features -experimental support for versions 3.2 and 3.3. +experimental support for versions from 3.2.3 to 3.3. Get Python at http://www.python.org. If you're running Linux or Mac OS X, you probably already have it installed. From 92fc263a2898b804c3b46447fd47e2898fbf8ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Fri, 2 Nov 2012 14:54:50 +0200 Subject: [PATCH 061/302] Fixed a regression in gis introduced by Query.select_fields removal --- django/contrib/gis/db/models/query.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django/contrib/gis/db/models/query.py b/django/contrib/gis/db/models/query.py index 2ffbd2021b..c89912b2d9 100644 --- a/django/contrib/gis/db/models/query.py +++ b/django/contrib/gis/db/models/query.py @@ -760,8 +760,10 @@ class GeoQuerySet(QuerySet): self.query.add_select_related([field_name]) compiler = self.query.get_compiler(self.db) compiler.pre_sql_setup() - rel_table, rel_col = self.query.related_select_cols[self.query.related_select_fields.index(geo_field)] - return compiler._field_column(geo_field, rel_table) + for (rel_table, rel_col), field in self.query.related_select_cols: + if field == geo_field: + return compiler._field_column(geo_field, rel_table) + raise ValueError("%r not in self.query.related_select_cols" % geo_field) elif not geo_field in opts.local_fields: # This geographic field is inherited from another model, so we have to # use the db table for the _parent_ model instead. From feaf9f279a73d87549c17fc7fb36463f1c7367a1 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 2 Nov 2012 06:54:00 -0400 Subject: [PATCH 062/302] Fixed #15361 - Documented performance considerations for QuerySet.get() Thanks mmcnickle for the patch. --- docs/topics/db/optimization.txt | 35 ++++++++++++++++++++++++++++++++ tests/modeltests/basic/tests.py | 36 ++++++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/docs/topics/db/optimization.txt b/docs/topics/db/optimization.txt index 772792d39d..b5cca52e23 100644 --- a/docs/topics/db/optimization.txt +++ b/docs/topics/db/optimization.txt @@ -132,6 +132,41 @@ Write your own :doc:`custom SQL to retrieve data or populate models `. Use ``django.db.connection.queries`` to find out what Django is writing for you and start from there. +Retrieve individual objects using a unique, indexed column +========================================================== + +There are two reasons to use a column with +:attr:`~django.db.models.Field.unique` or +:attr:`~django.db.models.Field.db_index` when using +:meth:`~django.db.models.query.QuerySet.get` to retrieve individual objects. +First, the query will be quicker because of the underlying database index. +Also, the query could run much slower if multiple objects match the lookup; +having a unique constraint on the column guarantees this will never happen. + +So using the :ref:`example Weblog models `:: + + >>> entry = Entry.objects.get(id=10) + +will be quicker than: + + >>> entry = Entry.object.get(headline="News Item Title") + +because ``id`` is indexed by the database and is guaranteed to be unique. + +Doing the following is potentially quite slow: + + >>> entry = Entry.objects.get(headline__startswith="News") + +First of all, `headline` is not indexed, which will make the underlying +database fetch slower. + +Second, the lookup doesn't guarantee that only one object will be returned. +If the query matches more than one object, it will retrieve and transfer all of +them from the database. This penalty could be substantial if hundreds or +thousands of records are returned. The penalty will be compounded if the +database lives on a separate server, where network overhead and latency also +play a factor. + Retrieve everything at once if you know you will need it ======================================================== diff --git a/tests/modeltests/basic/tests.py b/tests/modeltests/basic/tests.py index ebd70d14d9..1c83b980a7 100644 --- a/tests/modeltests/basic/tests.py +++ b/tests/modeltests/basic/tests.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, unicode_literals from datetime import datetime -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models.fields import Field, FieldDoesNotExist from django.test import TestCase, skipIfDBFeature, skipUnlessDBFeature from django.utils import six @@ -128,6 +128,40 @@ class ModelTest(TestCase): b = Article.objects.get(pk=a.id) self.assertEqual(a, b) + # Create a very similar object + a = Article( + id=None, + headline='Area man programs in Python', + pub_date=datetime(2005, 7, 28), + ) + a.save() + + self.assertEqual(Article.objects.count(), 2) + + # Django raises an Article.MultipleObjectsReturned exception if the + # lookup matches more than one object + self.assertRaisesRegexp( + MultipleObjectsReturned, + "get\(\) returned more than one Article -- it returned 2!", + Article.objects.get, + headline__startswith='Area', + ) + + self.assertRaisesRegexp( + MultipleObjectsReturned, + "get\(\) returned more than one Article -- it returned 2!", + Article.objects.get, + pub_date__year=2005, + ) + + self.assertRaisesRegexp( + MultipleObjectsReturned, + "get\(\) returned more than one Article -- it returned 2!", + Article.objects.get, + pub_date__year=2005, + pub_date__month=7, + ) + def test_object_creation(self): # Create an Article. a = Article( From 082fad0b83638332f85c5957eb8dcc5e38417608 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 2 Nov 2012 16:15:40 -0400 Subject: [PATCH 063/302] Cleaned up contrib.admin install instructions. Thanks Cal Leeming for the patch. --- docs/ref/contrib/admin/index.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index 6ed929cb7d..b661806c76 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -26,9 +26,10 @@ There are seven steps in activating the Django admin site: in your :setting:`INSTALLED_APPS` list, add them. 3. Add ``django.contrib.messages.context_processors.messages`` to - :setting:`TEMPLATE_CONTEXT_PROCESSORS` and - :class:`~django.contrib.messages.middleware.MessageMiddleware` to - :setting:`MIDDLEWARE_CLASSES`. (These are both active by default, so + :setting:`TEMPLATE_CONTEXT_PROCESSORS` as well as + :class:`django.contrib.auth.middleware.AuthenticationMiddleware` and + :class:`django.contrib.messages.middleware.MessageMiddleware` to + :setting:`MIDDLEWARE_CLASSES`. (These are all active by default, so you only need to do this if you've manually tweaked the settings.) 4. Determine which of your application's models should be editable in the From 07361d1fd6b4531e422e2593c91b47bc6bf88993 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 2 Nov 2012 16:33:07 -0400 Subject: [PATCH 064/302] Fixed #19167 - Added a warning regarding module-level database queries Thanks Daniele Procida for the patch. --- docs/topics/testing.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/topics/testing.txt b/docs/topics/testing.txt index 8fccf32946..a1524e4f15 100644 --- a/docs/topics/testing.txt +++ b/docs/topics/testing.txt @@ -379,6 +379,15 @@ control the particular collation used by the test database. See the :doc:`settings documentation ` for details of these advanced settings. +.. admonition:: Finding data from your production database when running tests? + + If your code attempts to access the database when its modules are compiled, + this will occur *before* the test database is set up, with potentially + unexpected results. For example, if you have a database query in + module-level code and a real database exists, production data could pollute + your tests. *It is a bad idea to have such import-time database queries in + your code* anyway - rewrite your code so that it doesn't do this. + .. _topics-testing-masterslave: Testing master/slave configurations From d1de7596b207b3a7dea8203334ef1739db3b1c94 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Fri, 2 Nov 2012 16:48:55 -0400 Subject: [PATCH 065/302] Fixed #19120 - Added an example of using ModelAdmin methods for read-only fields. Thanks Daniele Procida for the patch. --- docs/ref/contrib/admin/index.txt | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index b661806c76..f6da5b6cb2 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -816,15 +816,34 @@ subclass:: By default the admin shows all fields as editable. Any fields in this option (which should be a ``list`` or ``tuple``) will display its data - as-is and non-editable. This option behaves nearly identical to - :attr:`ModelAdmin.list_display`. Usage is the same, however, when you - specify :attr:`ModelAdmin.fields` or :attr:`ModelAdmin.fieldsets` the - read-only fields must be present to be shown (they are ignored otherwise). + as-is and non-editable. Note that when specifying :attr:`ModelAdmin.fields` + or :attr:`ModelAdmin.fieldsets` the read-only fields must be present to be + shown (they are ignored otherwise). If ``readonly_fields`` is used without defining explicit ordering through :attr:`ModelAdmin.fields` or :attr:`ModelAdmin.fieldsets` they will be added last after all editable fields. + A read-only field can not only display data from a model's field, it can + also display the output of a a model's method or a method of the + ``ModelAdmin`` class itself. This is very similar to the way + :attr:`ModelAdmin.list_display` behaves. This provides an easy way to use + the admin interface to provide feedback on the status of the objects being + edited, for example:: + + class PersonAdmin(ModelAdmin): + readonly_fields = ('address_report',) + + def address_report(self, instance): + return ", ".join(instance.get_full_address()) or \ + "I can't determine this address." + + # short_description functions like a model field's verbose_name + address_report.short_description = "Address" + # in this example, we have used HTML tags in the output + address_report.allow_tags = True + + .. attribute:: ModelAdmin.save_as Set ``save_as`` to enable a "save as" feature on admin change forms. From 965cc0b1ffa27574e5684a18f9400577e5b4598d Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Fri, 2 Nov 2012 15:49:29 +0000 Subject: [PATCH 066/302] Deprecated depth kwarg on select_related. This is the start of a deprecation path for the depth kwarg on select_related. Removing this will allow us to update select_related so it chains properly and have an API similar to prefetch_related. Thanks to Marc Tamlyn for spearheading and initial patch. refs #16855 --- django/db/models/query.py | 4 ++++ docs/internals/deprecation.txt | 3 +++ docs/ref/models/querysets.txt | 30 ++++++++++++++++-------------- docs/releases/1.5.txt | 7 +++++++ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/django/db/models/query.py b/django/db/models/query.py index da4c69f362..c00080abef 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -5,6 +5,7 @@ The main QuerySet implementation. This provides the public API for the ORM. import copy import itertools import sys +import warnings from django.core import exceptions from django.db import connections, router, transaction, IntegrityError @@ -698,6 +699,9 @@ class QuerySet(object): If fields are specified, they must be ForeignKey fields and only those related objects are included in the selection. """ + if 'depth' in kwargs: + warnings.warn('The "depth" keyword argument has been deprecated.\n' + 'Use related field names instead.', PendingDeprecationWarning) depth = kwargs.pop('depth', 0) if kwargs: raise TypeError('Unexpected keyword arguments to select_related: %s' diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 77371c8608..9fd92db2b4 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -298,6 +298,9 @@ these changes. * The ``daily_cleanup.py`` script will be removed. +* The ``depth`` keyword argument will be removed from + :meth:`~django.db.models.query.QuerySet.select_related`. + 2.0 --- diff --git a/docs/ref/models/querysets.txt b/docs/ref/models/querysets.txt index 7138cd0e74..295c996af4 100644 --- a/docs/ref/models/querysets.txt +++ b/docs/ref/models/querysets.txt @@ -676,21 +676,12 @@ Note that, by default, ``select_related()`` does not follow foreign keys that have ``null=True``. Usually, using ``select_related()`` can vastly improve performance because your -app can avoid many database calls. However, in situations with deeply nested -sets of relationships ``select_related()`` can sometimes end up following "too -many" relations, and can generate queries so large that they end up being slow. +app can avoid many database calls. However, there are times you are only +interested in specific related models, or have deeply nested sets of +relationships, and in these cases ``select_related()`` can can be optimized by +explicitly passing the related field names you are interested in. Only +the specified relations will be followed. -In these situations, you can use the ``depth`` argument to ``select_related()`` -to control how many "levels" of relations ``select_related()`` will actually -follow:: - - b = Book.objects.select_related(depth=1).get(id=4) - p = b.author # Doesn't hit the database. - c = p.hometown # Requires a database call. - -Sometimes you only want to access specific models that are related to your root -model, not all of the related models. In these cases, you can pass the related -field names to ``select_related()`` and it will only follow those relations. You can even do this for models that are more than one relation away by separating the field names with double underscores, just as for filters. For example, if you have this model:: @@ -730,6 +721,17 @@ You can also refer to the reverse direction of a is defined. Instead of specifying the field name, use the :attr:`related_name ` for the field on the related object. +.. deprecated:: 1.5 + The ``depth`` parameter to ``select_related()`` has been deprecated. You + should replace it with the use of the ``(*fields)`` listing specific + related fields instead as documented above. + +A depth limit of relationships to follow can also be specified:: + + b = Book.objects.select_related(depth=1).get(id=4) + p = b.author # Doesn't hit the database. + c = p.hometown # Requires a database call. + A :class:`~django.db.models.OneToOneField` is not traversed in the reverse direction if you are performing a depth-based ``select_related()`` call. diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index e18a78afc8..154f711560 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -638,3 +638,10 @@ The :djadmin:`cleanup` management command has been deprecated and replaced by The undocumented ``daily_cleanup.py`` script has been deprecated. Use the :djadmin:`clearsessions` management command instead. + +``depth`` keyword argument in ``select_related`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``depth`` keyword argument in +:meth:`~django.db.models.query.QuerySet.select_related` has been deprecated. +You should use field names instead. From 39f5bc7fc3a4bb43ed8a1358b17fe0521a1a63ac Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 3 Nov 2012 05:22:34 -0400 Subject: [PATCH 067/302] Fixed #16841 - Documented a couple ModelAdmin methods * ModelAdmin.get_changelist_form and get_changelist_formset * InlineModelAdmin.get_formset Thanks Jordan Reiter for the report. --- docs/ref/contrib/admin/index.txt | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index f6da5b6cb2..ee1342b43d 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -1233,10 +1233,39 @@ templates used by the :class:`ModelAdmin` views: .. method:: ModelAdmin.get_changelist(self, request, **kwargs) - Returns the Changelist class to be used for listing. By default, + Returns the ``Changelist`` class to be used for listing. By default, ``django.contrib.admin.views.main.ChangeList`` is used. By inheriting this class you can change the behavior of the listing. +.. method:: ModelAdmin.get_changelist_form(self, request, **kwargs) + + Returns a :class:`~django.forms.ModelForm` class for use in the ``Formset`` + on the changelist page. To use a custom form, for example:: + + class MyForm(forms.ModelForm): + class Meta: + model = MyModel + + class MyModelAdmin(admin.ModelAdmin): + def get_changelist_form(self, request, **kwargs): + return MyForm + +.. method:: ModelAdmin.get_changelist_formset(self, request, **kwargs) + + Returns a :ref:`ModelFormSet ` class for use on the + changelist page if :attr:`~ModelAdmin.list_editable` is used. To use a + custom formset, for example:: + + from django.forms.models import BaseModelFormSet + + class MyAdminFormSet(BaseModelFormSet): + pass + + class MyModelAdmin(admin.ModelAdmin): + def get_changelist_formset(self, request, **kwargs): + kwargs['formset'] = MyAdminFormSet + return super(MyModelAdmin, self).get_changelist_formset(request, **kwargs) + .. method:: ModelAdmin.has_add_permission(self, request) Should return ``True`` if adding an object is permitted, ``False`` @@ -1552,6 +1581,10 @@ The ``InlineModelAdmin`` class adds: Specifies whether or not inline objects can be deleted in the inline. Defaults to ``True``. +.. method:: InlineModelAdmin.get_formset(self, request, obj=None, **kwargs) + + Returns a ``BaseInlineFormSet`` class for use in admin add/change views. + See the example for :class:`ModelAdmin.get_formsets`. Working with a model with two or more foreign keys to the same parent model --------------------------------------------------------------------------- From ac2052ebc84c45709ab5f0f25e685bf656ce79bc Mon Sep 17 00:00:00 2001 From: Ulrich Petri Date: Sat, 7 Jul 2012 14:24:50 +0200 Subject: [PATCH 068/302] Fixed #17549 -- Added a clickable link for URLFields in admin change list. --- .../admin/static/admin/css/widgets.css | 15 +++++++++ django/contrib/admin/widgets.py | 15 ++++++++- docs/ref/models/fields.txt | 5 +++ tests/regressiontests/admin_widgets/tests.py | 31 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/django/contrib/admin/static/admin/css/widgets.css b/django/contrib/admin/static/admin/css/widgets.css index 0a7012c7b2..3b19353e6f 100644 --- a/django/contrib/admin/static/admin/css/widgets.css +++ b/django/contrib/admin/static/admin/css/widgets.css @@ -225,6 +225,21 @@ table p.datetime { padding-left: 0; } +/* URL */ + +p.url { + line-height: 20px; + margin: 0; + padding: 0; + color: #666; + font-size: 11px; + font-weight: bold; +} + +.url a { + font-weight: normal; +} + /* FILE UPLOADS */ p.file-upload { diff --git a/django/contrib/admin/widgets.py b/django/contrib/admin/widgets.py index 1e0bc2d366..1e6277fb87 100644 --- a/django/contrib/admin/widgets.py +++ b/django/contrib/admin/widgets.py @@ -10,7 +10,7 @@ from django.contrib.admin.templatetags.admin_static import static from django.core.urlresolvers import reverse from django.forms.widgets import RadioFieldRenderer from django.forms.util import flatatt -from django.utils.html import escape, format_html, format_html_join +from django.utils.html import escape, format_html, format_html_join, smart_urlquote from django.utils.text import Truncator from django.utils.translation import ugettext as _ from django.utils.safestring import mark_safe @@ -306,6 +306,19 @@ class AdminURLFieldWidget(forms.TextInput): final_attrs.update(attrs) super(AdminURLFieldWidget, self).__init__(attrs=final_attrs) + def render(self, name, value, attrs=None): + html = super(AdminURLFieldWidget, self).render(name, value, attrs) + if value: + value = force_text(self._format_value(value)) + final_attrs = {'href': mark_safe(smart_urlquote(value))} + html = format_html( + '

{0} {2}
{3} {4}

', + _('Currently:'), flatatt(final_attrs), value, + _('Change:'), html + ) + return html + + class AdminIntegerFieldWidget(forms.TextInput): class_name = 'vIntegerField' diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index 809d56eaf5..d8ea6bb31d 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -922,6 +922,11 @@ Like all :class:`CharField` subclasses, :class:`URLField` takes the optional :attr:`~CharField.max_length`argument. If you don't specify :attr:`~CharField.max_length`, a default of 200 is used. +.. versionadded:: 1.5 + +The current value of the field will be displayed as a clickable link above the +input widget. + Relationship fields =================== diff --git a/tests/regressiontests/admin_widgets/tests.py b/tests/regressiontests/admin_widgets/tests.py index 4b115431c1..0b016d885b 100644 --- a/tests/regressiontests/admin_widgets/tests.py +++ b/tests/regressiontests/admin_widgets/tests.py @@ -266,6 +266,37 @@ class AdminSplitDateTimeWidgetTest(DjangoTestCase): ) +class AdminURLWidgetTest(DjangoTestCase): + def test_render(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', '')), + '' + ) + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example.com')), + '

Currently:http://example.com
Change:

' + ) + + def test_render_idn(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example-äüö.com')), + '

Currently:http://example-äüö.com
Change:

' + ) + + def test_render_quoting(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example.com/some text')), + '

Currently:http://example.com/<sometag>some text</sometag>
Change:

' + ) + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example-äüö.com/some text')), + '

Currently:http://example-äüö.com/<sometag>some text</sometag>
Change:

' + ) + + class AdminFileWidgetTest(DjangoTestCase): def test_render(self): band = models.Band.objects.create(name='Linkin Park') From 095eca8dd85cb27ed0b22829903df10f19cdab6c Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 3 Nov 2012 12:54:06 +0100 Subject: [PATCH 069/302] Fixed #19101 -- Decoding of non-ASCII POST data on Python 3. Thanks Claude Paroz. --- django/http/multipartparser.py | 2 +- django/http/request.py | 3 +++ tests/regressiontests/requests/tests.py | 12 +++++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 40aefd6e9d..5bcc874982 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -110,7 +110,7 @@ class MultiPartParser(object): # HTTP spec says that Content-Length >= 0 is valid # handling content-length == 0 before continuing if self._content_length == 0: - return QueryDict(MultiValueDict(), encoding=self._encoding), MultiValueDict() + return QueryDict('', encoding=self._encoding), MultiValueDict() # See if the handler will want to take care of the parsing. # This allows overriding everything if somebody wants it. diff --git a/django/http/request.py b/django/http/request.py index 96c7606c86..d3f0888d47 100644 --- a/django/http/request.py +++ b/django/http/request.py @@ -276,6 +276,9 @@ class QueryDict(MultiValueDict): encoding = settings.DEFAULT_CHARSET self.encoding = encoding if six.PY3: + if isinstance(query_string, bytes): + # query_string contains URL-encoded data, a subset of ASCII. + query_string = query_string.decode() for key, value in parse_qsl(query_string or '', keep_blank_values=True, encoding=encoding): diff --git a/tests/regressiontests/requests/tests.py b/tests/regressiontests/requests/tests.py index eaf25ea7a6..164c1082fe 100644 --- a/tests/regressiontests/requests/tests.py +++ b/tests/regressiontests/requests/tests.py @@ -12,7 +12,7 @@ from django.http import HttpRequest, HttpResponse, parse_cookie, build_request_r from django.test.client import FakePayload from django.test.utils import override_settings, str_prefix from django.utils import unittest -from django.utils.http import cookie_date +from django.utils.http import cookie_date, urlencode from django.utils.timezone import utc @@ -353,6 +353,16 @@ class RequestsTests(unittest.TestCase): self.assertRaises(Exception, lambda: request.body) self.assertEqual(request.POST, {}) + def test_non_ascii_POST(self): + payload = FakePayload(urlencode({'key': 'España'})) + request = WSGIRequest({ + 'REQUEST_METHOD': 'POST', + 'CONTENT_LENGTH': len(payload), + 'CONTENT_TYPE': 'application/x-www-form-urlencoded', + 'wsgi.input': payload, + }) + self.assertEqual(request.POST, {'key': ['España']}) + def test_alternate_charset_POST(self): """ Test a POST with non-utf-8 payload encoding. From 0546794397130b1574a667d57667bd032bff78d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Zapke-Gr=C3=BCndemann?= Date: Sat, 3 Nov 2012 17:04:53 +0100 Subject: [PATCH 070/302] Fixed #19230 -- Extended the handler403 documentation. Added a paragraph on how to use the PermissionDenied exception to create a 403 response and use handler403. --- docs/topics/http/views.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/topics/http/views.txt b/docs/topics/http/views.txt index 7c4d1bbb6e..caa2882f37 100644 --- a/docs/topics/http/views.txt +++ b/docs/topics/http/views.txt @@ -209,6 +209,17 @@ This view loads and renders the template ``403.html`` in your root template directory, or if this file does not exist, instead serves the text "403 Forbidden", as per :rfc:`2616` (the HTTP 1.1 Specification). +``django.views.defaults.permission_denied`` is triggered by a +:exc:`~django.core.exceptions.PermissionDenied` exception. To deny access in a +view you can use code like this:: + + from django.core.exceptions import PermissionDenied + + def edit(request, pk): + if not request.user.is_staff: + raise PermissionDenied + # ... + It is possible to override ``django.views.defaults.permission_denied`` in the same way you can for the 404 and 500 views by specifying a ``handler403`` in your URLconf:: From 3e98d98b69e67f2f72055e4b3204d0486eaeff50 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 3 Nov 2012 20:07:56 +0100 Subject: [PATCH 071/302] Prevented host resolution when running dev server Refs #19075, #2494. Thanks Karen Tracey for spotting the issue. --- django/core/servers/basehttp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django/core/servers/basehttp.py b/django/core/servers/basehttp.py index a7004f2c2f..7387d13199 100644 --- a/django/core/servers/basehttp.py +++ b/django/core/servers/basehttp.py @@ -138,6 +138,10 @@ class WSGIRequestHandler(simple_server.WSGIRequestHandler, object): self.style = color_style() super(WSGIRequestHandler, self).__init__(*args, **kwargs) + def address_string(self): + # Short-circuit parent method to not call socket.getfqdn + return self.client_address[0] + def log_message(self, format, *args): # Don't bother logging requests for admin images or the favicon. if (self.path.startswith(self.admin_static_prefix) From 90e530978d590a5bdcf75525aa03f844766018b8 Mon Sep 17 00:00:00 2001 From: Gabriel Hurley Date: Sat, 3 Nov 2012 13:06:57 -0700 Subject: [PATCH 072/302] Fixed #18210 -- Escaped special characters in reverse prefixes. Ensured that special characters passed in to reverse via the prefix argument are properly escaped so that calls to django.utils.regex_helpers.normalize and/or string formatting operations don't result in exceptions. Thanks to toofishes for the error report. --- django/core/urlresolvers.py | 10 ++++++---- tests/regressiontests/urlpatterns_reverse/tests.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/django/core/urlresolvers.py b/django/core/urlresolvers.py index af3df83d0a..9c0b7b70fd 100644 --- a/django/core/urlresolvers.py +++ b/django/core/urlresolvers.py @@ -16,6 +16,7 @@ from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist from django.utils.datastructures import MultiValueDict from django.utils.encoding import force_str, force_text, iri_to_uri from django.utils.functional import memoize, lazy +from django.utils.http import urlquote from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule from django.utils.regex_helper import normalize @@ -379,14 +380,15 @@ class RegexURLResolver(LocaleRegexProvider): except (ImportError, AttributeError) as e: raise NoReverseMatch("Error importing '%s': %s." % (lookup_view, e)) possibilities = self.reverse_dict.getlist(lookup_view) - prefix_norm, prefix_args = normalize(_prefix)[0] + + prefix_norm, prefix_args = normalize(urlquote(_prefix))[0] for possibility, pattern, defaults in possibilities: for result, params in possibility: if args: if len(args) != len(params) + len(prefix_args): continue unicode_args = [force_text(val) for val in args] - candidate = (prefix_norm + result) % dict(zip(prefix_args + params, unicode_args)) + candidate = (prefix_norm + result) % dict(zip(prefix_args + params, unicode_args)) else: if set(kwargs.keys()) | set(defaults.keys()) != set(params) | set(defaults.keys()) | set(prefix_args): continue @@ -398,8 +400,8 @@ class RegexURLResolver(LocaleRegexProvider): if not matches: continue unicode_kwargs = dict([(k, force_text(v)) for (k, v) in kwargs.items()]) - candidate = (prefix_norm + result) % unicode_kwargs - if re.search('^%s%s' % (_prefix, pattern), candidate, re.UNICODE): + candidate = (prefix_norm.replace('%', '%%') + result) % unicode_kwargs + if re.search('^%s%s' % (prefix_norm, pattern), candidate, re.UNICODE): return candidate # lookup_view can be URL label, or dotted path, or callable, Any of # these can be passed in at the top, but callables are not friendly in diff --git a/tests/regressiontests/urlpatterns_reverse/tests.py b/tests/regressiontests/urlpatterns_reverse/tests.py index 234897d267..85f18db4c5 100644 --- a/tests/regressiontests/urlpatterns_reverse/tests.py +++ b/tests/regressiontests/urlpatterns_reverse/tests.py @@ -171,6 +171,18 @@ class URLPatternReverse(TestCase): # Reversing None should raise an error, not return the last un-named view. self.assertRaises(NoReverseMatch, reverse, None) + def test_prefix_braces(self): + self.assertEqual('/%7B%7Binvalid%7D%7D/includes/non_path_include/', + reverse('non_path_include', prefix='/{{invalid}}/')) + + def test_prefix_parenthesis(self): + self.assertEqual('/bogus%29/includes/non_path_include/', + reverse('non_path_include', prefix='/bogus)/')) + + def test_prefix_format_char(self): + self.assertEqual('/bump%2520map/includes/non_path_include/', + reverse('non_path_include', prefix='/bump%20map/')) + class ResolverTests(unittest.TestCase): def test_resolver_repr(self): """ From 973f539ab83bb46645f2f711190735c66a246797 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 3 Nov 2012 21:26:59 +0100 Subject: [PATCH 073/302] Fixed #15152 -- Avoided crash of CommonMiddleware on broken querystring --- django/middleware/common.py | 13 ++++++++++++- tests/regressiontests/middleware/tests.py | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/django/middleware/common.py b/django/middleware/common.py index 6fbbf43044..ccc9fbfaad 100644 --- a/django/middleware/common.py +++ b/django/middleware/common.py @@ -6,6 +6,7 @@ from django.conf import settings from django import http from django.core.mail import mail_managers from django.utils.http import urlquote +from django.utils import six from django.core import urlresolvers @@ -87,7 +88,17 @@ class CommonMiddleware(object): else: newurl = urlquote(new_url[1]) if request.META.get('QUERY_STRING', ''): - newurl += '?' + request.META['QUERY_STRING'] + if six.PY3: + newurl += '?' + request.META['QUERY_STRING'] + else: + # `query_string` is a bytestring. Appending it to the unicode + # string `newurl` will fail if it isn't ASCII-only. This isn't + # allowed; only broken software generates such query strings. + # Better drop the invalid query string than crash (#15152). + try: + newurl += '?' + request.META['QUERY_STRING'].decode() + except UnicodeDecodeError: + pass return http.HttpResponsePermanentRedirect(newurl) def process_response(self, request, response): diff --git a/tests/regressiontests/middleware/tests.py b/tests/regressiontests/middleware/tests.py index de901f4a80..b8cffd9c92 100644 --- a/tests/regressiontests/middleware/tests.py +++ b/tests/regressiontests/middleware/tests.py @@ -294,6 +294,15 @@ class CommonMiddlewareTest(TestCase): CommonMiddleware().process_response(request, response) self.assertEqual(len(mail.outbox), 0) + # Other tests + + def test_non_ascii_query_string_does_not_crash(self): + """Regression test for #15152""" + request = self._get_request('slash') + request.META['QUERY_STRING'] = 'drink=café' + response = CommonMiddleware().process_request(request) + self.assertEqual(response.status_code, 301) + class ConditionalGetMiddlewareTest(TestCase): urls = 'regressiontests.middleware.cond_get_urls' From fc10418fba4fb906e4265650b62c510d526d63f7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 3 Nov 2012 21:43:11 +0100 Subject: [PATCH 074/302] Fixed #18963 -- Used a subclass-friendly pattern for Python 2 object model compatibility methods. --- django/contrib/auth/context_processors.py | 4 +++- django/contrib/gis/measure.py | 16 ++++++++++++---- django/core/files/base.py | 8 ++++++-- django/core/serializers/base.py | 4 +--- django/core/serializers/xml_serializer.py | 2 -- django/db/backends/oracle/base.py | 4 +--- django/db/models/expressions.py | 12 ++++++++---- django/db/models/query.py | 4 +++- django/dispatch/saferef.py | 16 +++++++++------- django/forms/formsets.py | 4 +++- django/http/multipartparser.py | 16 ++++------------ django/http/response.py | 4 +--- django/utils/tree.py | 4 +++- docs/topics/python3.txt | 13 +++++++------ 14 files changed, 61 insertions(+), 50 deletions(-) diff --git a/django/contrib/auth/context_processors.py b/django/contrib/auth/context_processors.py index 5929505359..3d17fe2754 100644 --- a/django/contrib/auth/context_processors.py +++ b/django/contrib/auth/context_processors.py @@ -18,7 +18,9 @@ class PermLookupDict(object): def __bool__(self): return self.user.has_module_perms(self.module_name) - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) class PermWrapper(object): diff --git a/django/contrib/gis/measure.py b/django/contrib/gis/measure.py index 6e074be355..e2e6b6bca8 100644 --- a/django/contrib/gis/measure.py +++ b/django/contrib/gis/measure.py @@ -151,7 +151,9 @@ class MeasureBase(object): **{self.STANDARD_UNIT: (self.standard / other)}) else: raise TypeError('%(class)s must be divided with number or %(class)s' % {"class":pretty_name(self)}) - __div__ = __truediv__ # Python 2 compatibility + + def __div__(self, other): # Python 2 compatibility + return type(self).__truediv__(self, other) def __itruediv__(self, other): if isinstance(other, NUMERIC_TYPES): @@ -159,11 +161,15 @@ class MeasureBase(object): return self else: raise TypeError('%(class)s must be divided with number' % {"class":pretty_name(self)}) - __idiv__ = __itruediv__ # Python 2 compatibility + + def __idiv__(self, other): # Python 2 compatibility + return type(self).__itruediv__(self, other) def __bool__(self): return bool(self.standard) - __nonzero__ = __bool__ # Python 2 compatibility + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def default_units(self, kwargs): """ @@ -314,7 +320,9 @@ class Area(MeasureBase): **{self.STANDARD_UNIT: (self.standard / other)}) else: raise TypeError('%(class)s must be divided by a number' % {"class":pretty_name(self)}) - __div__ = __truediv__ # Python 2 compatibility + + def __div__(self, other): # Python 2 compatibility + return type(self).__truediv__(self, other) # Shortcuts diff --git a/django/core/files/base.py b/django/core/files/base.py index b81e180292..71de5ab741 100644 --- a/django/core/files/base.py +++ b/django/core/files/base.py @@ -28,7 +28,9 @@ class File(FileProxyMixin): def __bool__(self): return bool(self.name) - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def __len__(self): return self.size @@ -142,7 +144,9 @@ class ContentFile(File): def __bool__(self): return True - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def open(self, mode=None): self.seek(0) diff --git a/django/core/serializers/base.py b/django/core/serializers/base.py index 276f9a4738..294934a04a 100644 --- a/django/core/serializers/base.py +++ b/django/core/serializers/base.py @@ -112,7 +112,7 @@ class Serializer(object): if callable(getattr(self.stream, 'getvalue', None)): return self.stream.getvalue() -class Deserializer(object): +class Deserializer(six.Iterator): """ Abstract base deserializer class. """ @@ -138,8 +138,6 @@ class Deserializer(object): """Iteration iterface -- return the next item in the stream""" raise NotImplementedError - next = __next__ # Python 2 compatibility - class DeserializedObject(object): """ A deserialized model. diff --git a/django/core/serializers/xml_serializer.py b/django/core/serializers/xml_serializer.py index 666587dc77..ea333a22bd 100644 --- a/django/core/serializers/xml_serializer.py +++ b/django/core/serializers/xml_serializer.py @@ -161,8 +161,6 @@ class Deserializer(base.Deserializer): return self._handle_object(node) raise StopIteration - next = __next__ # Python 2 compatibility - def _handle_object(self, node): """ Convert an node to a DeserializedObject. diff --git a/django/db/backends/oracle/base.py b/django/db/backends/oracle/base.py index aad52992dd..dfdfd4fd49 100644 --- a/django/db/backends/oracle/base.py +++ b/django/db/backends/oracle/base.py @@ -774,7 +774,7 @@ class FormatStylePlaceholderCursor(object): return CursorIterator(self.cursor) -class CursorIterator(object): +class CursorIterator(six.Iterator): """Cursor iterator wrapper that invokes our custom row factory.""" @@ -788,8 +788,6 @@ class CursorIterator(object): def __next__(self): return _rowfactory(next(self.iter), self.cursor) - next = __next__ # Python 2 compatibility - def _rowfactory(row, cursor): # Cast numeric values as the appropriate Python type based upon the diff --git a/django/db/models/expressions.py b/django/db/models/expressions.py index 30c44bacde..3566d777c6 100644 --- a/django/db/models/expressions.py +++ b/django/db/models/expressions.py @@ -62,7 +62,9 @@ class ExpressionNode(tree.Node): def __truediv__(self, other): return self._combine(other, self.DIV, False) - __div__ = __truediv__ # Python 2 compatibility + + def __div__(self, other): # Python 2 compatibility + return type(self).__truediv__(self, other) def __mod__(self, other): return self._combine(other, self.MOD, False) @@ -94,7 +96,9 @@ class ExpressionNode(tree.Node): def __rtruediv__(self, other): return self._combine(other, self.DIV, True) - __rdiv__ = __rtruediv__ # Python 2 compatibility + + def __rdiv__(self, other): # Python 2 compatibility + return type(self).__rtruediv__(self, other) def __rmod__(self, other): return self._combine(other, self.MOD, True) @@ -151,10 +155,10 @@ class DateModifierNode(ExpressionNode): (A custom function is used in order to preserve six digits of fractional second information on sqlite, and to format both date and datetime values.) - Note that microsecond comparisons are not well supported with MySQL, since + Note that microsecond comparisons are not well supported with MySQL, since MySQL does not store microsecond information. - Only adding and subtracting timedeltas is supported, attempts to use other + Only adding and subtracting timedeltas is supported, attempts to use other operations raise a TypeError. """ def __init__(self, children, connector, negated=False): diff --git a/django/db/models/query.py b/django/db/models/query.py index c00080abef..a3b28e9228 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -136,7 +136,9 @@ class QuerySet(object): except StopIteration: return False return True - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def __contains__(self, val): # The 'in' operator works without this method, due to __iter__. This diff --git a/django/dispatch/saferef.py b/django/dispatch/saferef.py index 84d1b2183c..7423669c96 100644 --- a/django/dispatch/saferef.py +++ b/django/dispatch/saferef.py @@ -67,9 +67,9 @@ class BoundMethodWeakref(object): same BoundMethodWeakref instance. """ - + _allInstances = weakref.WeakValueDictionary() - + def __new__( cls, target, onDelete=None, *arguments,**named ): """Create new instance or return current instance @@ -92,7 +92,7 @@ class BoundMethodWeakref(object): cls._allInstances[key] = base base.__init__( target, onDelete, *arguments,**named) return base - + def __init__(self, target, onDelete=None): """Return a weak-reference-like instance for a bound method @@ -132,7 +132,7 @@ class BoundMethodWeakref(object): self.weakFunc = weakref.ref(target.__func__, remove) self.selfName = str(target.__self__) self.funcName = str(target.__func__.__name__) - + def calculateKey( cls, target ): """Calculate the reference key for this reference @@ -141,7 +141,7 @@ class BoundMethodWeakref(object): """ return (id(target.__self__),id(target.__func__)) calculateKey = classmethod( calculateKey ) - + def __str__(self): """Give a friendly representation of the object""" return """%s( %s.%s )"""%( @@ -157,14 +157,16 @@ class BoundMethodWeakref(object): def __bool__( self ): """Whether we are still a valid reference""" return self() is not None - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def __eq__(self, other): """Compare with another reference""" if not isinstance(other, self.__class__): return self.__class__ == type(other) return self.key == other.key - + def __call__(self): """Return a strong reference to the bound method diff --git a/django/forms/formsets.py b/django/forms/formsets.py index c646eed506..a0a38f336f 100644 --- a/django/forms/formsets.py +++ b/django/forms/formsets.py @@ -69,7 +69,9 @@ class BaseFormSet(object): def __bool__(self): """All formsets have a management form which is not included in the length""" return True - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) @property def management_form(self): diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 5bcc874982..9413a1eabb 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -256,7 +256,7 @@ class MultiPartParser(object): """Cleanup filename from Internet Explorer full paths.""" return filename and filename[filename.rfind("\\")+1:].strip() -class LazyStream(object): +class LazyStream(six.Iterator): """ The LazyStream wrapper allows one to get and "unget" bytes from a stream. @@ -323,8 +323,6 @@ class LazyStream(object): self.position += len(output) return output - next = __next__ # Python 2 compatibility - def close(self): """ Used to invalidate/disable this lazy stream. @@ -369,7 +367,7 @@ class LazyStream(object): " if there is none, report this to the Django developers." ) -class ChunkIter(object): +class ChunkIter(six.Iterator): """ An iterable that will yield chunks of data. Given a file-like object as the constructor, this object will yield chunks of read operations from that @@ -389,12 +387,10 @@ class ChunkIter(object): else: raise StopIteration() - next = __next__ # Python 2 compatibility - def __iter__(self): return self -class InterBoundaryIter(object): +class InterBoundaryIter(six.Iterator): """ A Producer that will iterate over boundaries. """ @@ -411,9 +407,7 @@ class InterBoundaryIter(object): except InputStreamExhausted: raise StopIteration() - next = __next__ # Python 2 compatibility - -class BoundaryIter(object): +class BoundaryIter(six.Iterator): """ A Producer that is sensitive to boundaries. @@ -489,8 +483,6 @@ class BoundaryIter(object): stream.unget(chunk[-rollback:]) return chunk[:-rollback] - next = __next__ # Python 2 compatibility - def _find_boundary(self, data, eof = False): """ Finds a multipart boundary in data. diff --git a/django/http/response.py b/django/http/response.py index 56e3d00096..df0a955b18 100644 --- a/django/http/response.py +++ b/django/http/response.py @@ -23,7 +23,7 @@ class BadHeaderError(ValueError): pass -class HttpResponseBase(object): +class HttpResponseBase(six.Iterator): """ An HTTP response base class with dictionary-accessed headers. @@ -218,8 +218,6 @@ class HttpResponseBase(object): # Subclasses must define self._iterator for this function. return self.make_bytes(next(self._iterator)) - next = __next__ # Python 2 compatibility - # These methods partially implement the file-like object interface. # See http://docs.python.org/lib/bltin-file-objects.html diff --git a/django/utils/tree.py b/django/utils/tree.py index 717181d2b9..ce490224e0 100644 --- a/django/utils/tree.py +++ b/django/utils/tree.py @@ -73,7 +73,9 @@ class Node(object): For truth value testing. """ return bool(self.children) - __nonzero__ = __bool__ # Python 2 + + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) def __contains__(self, other): """ diff --git a/docs/topics/python3.txt b/docs/topics/python3.txt index f5749faaf2..e6dc165399 100644 --- a/docs/topics/python3.txt +++ b/docs/topics/python3.txt @@ -278,15 +278,13 @@ Iterators :: - class MyIterator(object): + class MyIterator(six.Iterator): def __iter__(self): return self # implement some logic here def __next__(self): raise StopIteration # implement some logic here - next = __next__ # Python 2 compatibility - Boolean evaluation ~~~~~~~~~~~~~~~~~~ @@ -297,7 +295,8 @@ Boolean evaluation def __bool__(self): return True # implement some logic here - __nonzero__ = __bool__ # Python 2 compatibility + def __nonzero__(self): # Python 2 compatibility + return type(self).__bool__(self) Division ~~~~~~~~ @@ -309,12 +308,14 @@ Division def __truediv__(self, other): return self / other # implement some logic here - __div__ = __truediv__ # Python 2 compatibility + def __div__(self, other): # Python 2 compatibility + return type(self).__truediv__(self, other) def __itruediv__(self, other): return self // other # implement some logic here - __idiv__ = __itruediv__ # Python 2 compatibility + def __idiv__(self, other): # Python 2 compatibility + return type(self).__itruediv__(self, other) .. module: django.utils.six From 4e8d9524c62d071718a85d1d55a18310be91b4f7 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 3 Nov 2012 23:53:51 +0100 Subject: [PATCH 075/302] Fixed #6234 -- Removed obsolete note about json and ensure_ascii Thanks aaron at cellmap.ca for the report. --- docs/topics/serialization.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/topics/serialization.txt b/docs/topics/serialization.txt index 9b44166e42..28f600e223 100644 --- a/docs/topics/serialization.txt +++ b/docs/topics/serialization.txt @@ -166,15 +166,6 @@ Notes for specific serialization formats json ^^^^ -If you're using UTF-8 (or any other non-ASCII encoding) data with the JSON -serializer, you must pass ``ensure_ascii=False`` as a parameter to the -``serialize()`` call. Otherwise, the output won't be encoded correctly. - -For example:: - - json_serializer = serializers.get_serializer("json")() - json_serializer.serialize(queryset, ensure_ascii=False, stream=response) - Be aware that not all Django output can be passed unmodified to :mod:`json`. In particular, :ref:`lazy translation objects ` need a `special encoder`_ written for them. Something like this will work:: From 249c3d730e632b3c5b8c2bf5e6e871d61df15c6c Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sun, 4 Nov 2012 05:32:53 -0500 Subject: [PATCH 076/302] Fixed #19090 - Added PostgreSQL connection note. Thanks Melevir for the patch. --- docs/ref/databases.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ref/databases.txt b/docs/ref/databases.txt index 3a52f838e7..946d0f4f3b 100644 --- a/docs/ref/databases.txt +++ b/docs/ref/databases.txt @@ -33,6 +33,15 @@ aggregate with a database backend that falls within the affected release range. .. _known to be faulty: http://archives.postgresql.org/pgsql-bugs/2007-07/msg00046.php .. _Release 8.2.5: http://www.postgresql.org/docs/devel/static/release-8-2-5.html +PostgreSQL connection settings +------------------------------ + +By default (empty :setting:`HOST`), the connection to the database is done +through UNIX domain sockets ('local' lines in pg_hba.conf). If you want to +connect through TCP sockets, set :setting:`HOST` to 'localhost' or '127.0.0.1' +('host' lines in pg_hba.conf). On Windows, you should always define +:setting:`HOST`, as UNIX domain sockets are not available. + Optimizing PostgreSQL's configuration ------------------------------------- From 4285571c5a9bf6ca3cb7c4d774942b9ae5b537e4 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 4 Nov 2012 10:16:06 -0800 Subject: [PATCH 077/302] Fixed #5805 -- it is now possible to specify multi-column indexes. Thanks to jgelens for the original patch. --- django/core/management/validation.py | 35 +++++++++---- django/db/backends/creation.py | 51 ++++++++++++------- django/db/models/options.py | 4 +- docs/ref/models/options.txt | 15 ++++++ .../invalid_models/invalid_models/models.py | 8 +++ tests/regressiontests/indexes/__init__.py | 0 tests/regressiontests/indexes/models.py | 11 ++++ tests/regressiontests/indexes/tests.py | 12 +++++ .../initial_sql_regress/tests.py | 7 ++- tests/regressiontests/introspection/models.py | 4 ++ tests/regressiontests/introspection/tests.py | 12 +++-- tests/runtests.py | 2 +- 12 files changed, 121 insertions(+), 40 deletions(-) create mode 100644 tests/regressiontests/indexes/__init__.py create mode 100644 tests/regressiontests/indexes/models.py create mode 100644 tests/regressiontests/indexes/tests.py diff --git a/django/core/management/validation.py b/django/core/management/validation.py index 957a712b72..32e7181dab 100644 --- a/django/core/management/validation.py +++ b/django/core/management/validation.py @@ -1,3 +1,4 @@ +import collections import sys from django.conf import settings @@ -327,15 +328,29 @@ def get_validation_errors(outfile, app=None): # Check unique_together. for ut in opts.unique_together: - for field_name in ut: - try: - f = opts.get_field(field_name, many_to_many=True) - except models.FieldDoesNotExist: - e.add(opts, '"unique_together" refers to %s, a field that doesn\'t exist. Check your syntax.' % field_name) - else: - if isinstance(f.rel, models.ManyToManyRel): - e.add(opts, '"unique_together" refers to %s. ManyToManyFields are not supported in unique_together.' % f.name) - if f not in opts.local_fields: - e.add(opts, '"unique_together" refers to %s. This is not in the same model as the unique_together statement.' % f.name) + validate_local_fields(e, opts, "unique_together", ut) + if not isinstance(opts.index_together, collections.Sequence): + e.add(opts, '"index_together" must a sequence') + else: + for it in opts.index_together: + validate_local_fields(e, opts, "index_together", it) return len(e.errors) + + +def validate_local_fields(e, opts, field_name, fields): + from django.db import models + + if not isinstance(fields, collections.Sequence): + e.add(opts, 'all %s elements must be sequences' % field_name) + else: + for field in fields: + try: + f = opts.get_field(field, many_to_many=True) + except models.FieldDoesNotExist: + e.add(opts, '"%s" refers to %s, a field that doesn\'t exist.' % (field_name, field)) + else: + if isinstance(f.rel, models.ManyToManyRel): + e.add(opts, '"%s" refers to %s. ManyToManyFields are not supported in %s.' % (field_name, f.name, field_name)) + if f not in opts.local_fields: + e.add(opts, '"%s" refers to %s. This is not in the same model as the %s statement.' % (field_name, f.name, field_name)) diff --git a/django/db/backends/creation.py b/django/db/backends/creation.py index 3262a8922f..4c4cf4d044 100644 --- a/django/db/backends/creation.py +++ b/django/db/backends/creation.py @@ -177,34 +177,47 @@ class BaseDatabaseCreation(object): output = [] for f in model._meta.local_fields: output.extend(self.sql_indexes_for_field(model, f, style)) + for fs in model._meta.index_together: + fields = [model._meta.get_field_by_name(f)[0] for f in fs] + output.extend(self.sql_indexes_for_fields(model, fields, style)) return output def sql_indexes_for_field(self, model, f, style): """ Return the CREATE INDEX SQL statements for a single model field. """ + if f.db_index and not f.unique: + return self.sql_indexes_for_fields(model, [f], style) + else: + return [] + + def sql_indexes_for_fields(self, model, fields, style): from django.db.backends.util import truncate_name - if f.db_index and not f.unique: - qn = self.connection.ops.quote_name - tablespace = f.db_tablespace or model._meta.db_tablespace - if tablespace: - tablespace_sql = self.connection.ops.tablespace_sql(tablespace) - if tablespace_sql: - tablespace_sql = ' ' + tablespace_sql - else: - tablespace_sql = '' - i_name = '%s_%s' % (model._meta.db_table, self._digest(f.column)) - output = [style.SQL_KEYWORD('CREATE INDEX') + ' ' + - style.SQL_TABLE(qn(truncate_name( - i_name, self.connection.ops.max_name_length()))) + ' ' + - style.SQL_KEYWORD('ON') + ' ' + - style.SQL_TABLE(qn(model._meta.db_table)) + ' ' + - "(%s)" % style.SQL_FIELD(qn(f.column)) + - "%s;" % tablespace_sql] + if len(fields) == 1 and fields[0].db_tablespace: + tablespace_sql = self.connection.ops.tablespace_sql(fields[0].db_tablespace) + elif model._meta.db_tablespace: + tablespace_sql = self.connection.ops.tablespace_sql(model._meta.db_tablespace) else: - output = [] - return output + tablespace_sql = "" + if tablespace_sql: + tablespace_sql = " " + tablespace_sql + + field_names = [] + qn = self.connection.ops.quote_name + for f in fields: + field_names.append(style.SQL_FIELD(qn(f.column))) + + index_name = "%s_%s" % (model._meta.db_table, self._digest([f.name for f in fields])) + + return [ + style.SQL_KEYWORD("CREATE INDEX") + " " + + style.SQL_TABLE(qn(truncate_name(index_name, self.connection.ops.max_name_length()))) + " " + + style.SQL_KEYWORD("ON") + " " + + style.SQL_TABLE(qn(model._meta.db_table)) + " " + + "(%s)" % style.SQL_FIELD(", ".join(field_names)) + + "%s;" % tablespace_sql, + ] def sql_destroy_model(self, model, references_to_delete, style): """ diff --git a/django/db/models/options.py b/django/db/models/options.py index f430caceef..b04f3d4c2d 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -21,7 +21,8 @@ get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]| DEFAULT_NAMES = ('verbose_name', 'verbose_name_plural', 'db_table', 'ordering', 'unique_together', 'permissions', 'get_latest_by', 'order_with_respect_to', 'app_label', 'db_tablespace', - 'abstract', 'managed', 'proxy', 'swappable', 'auto_created') + 'abstract', 'managed', 'proxy', 'swappable', 'auto_created', + 'index_together') @python_2_unicode_compatible @@ -34,6 +35,7 @@ class Options(object): self.db_table = '' self.ordering = [] self.unique_together = [] + self.index_together = [] self.permissions = [] self.object_name, self.app_label = None, app_label self.get_latest_by = None diff --git a/docs/ref/models/options.txt b/docs/ref/models/options.txt index c5ae8398ea..ab944d7dda 100644 --- a/docs/ref/models/options.txt +++ b/docs/ref/models/options.txt @@ -261,6 +261,21 @@ Django quotes column and table names behind the scenes. :class:`~django.db.models.ManyToManyField`, try using a signal or an explicit :attr:`through ` model. +``index_together`` + +.. versionadded:: 1.5 + +.. attribute:: Options.index_together + + Sets of field names that, taken together, are indexed:: + + index_together = [ + ["pub_date", "deadline"], + ] + + This list of fields will be indexed together (i.e. the appropriate + ``CREATE INDEX`` statement will be issued.) + ``verbose_name`` ---------------- diff --git a/tests/modeltests/invalid_models/invalid_models/models.py b/tests/modeltests/invalid_models/invalid_models/models.py index ccb6396352..3c21e1ddb8 100644 --- a/tests/modeltests/invalid_models/invalid_models/models.py +++ b/tests/modeltests/invalid_models/invalid_models/models.py @@ -356,6 +356,13 @@ class HardReferenceModel(models.Model): m2m_4 = models.ManyToManyField('invalid_models.SwappedModel', related_name='m2m_hardref4') +class BadIndexTogether1(models.Model): + class Meta: + index_together = [ + ["field_that_does_not_exist"], + ] + + model_errors = """invalid_models.fielderrors: "charfield": CharFields require a "max_length" attribute that is a positive integer. invalid_models.fielderrors: "charfield2": CharFields require a "max_length" attribute that is a positive integer. invalid_models.fielderrors: "charfield3": CharFields require a "max_length" attribute that is a positive integer. @@ -470,6 +477,7 @@ invalid_models.hardreferencemodel: 'm2m_3' defines a relation with the model 'in invalid_models.hardreferencemodel: 'm2m_4' defines a relation with the model 'invalid_models.SwappedModel', which has been swapped out. Update the relation to point at settings.TEST_SWAPPED_MODEL. invalid_models.badswappablevalue: TEST_SWAPPED_MODEL_BAD_VALUE is not of the form 'app_label.app_name'. invalid_models.badswappablemodel: Model has been swapped out for 'not_an_app.Target' which has not been installed or is abstract. +invalid_models.badindextogether1: "index_together" refers to field_that_does_not_exist, a field that doesn't exist. """ if not connection.features.interprets_empty_strings_as_nulls: diff --git a/tests/regressiontests/indexes/__init__.py b/tests/regressiontests/indexes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/regressiontests/indexes/models.py b/tests/regressiontests/indexes/models.py new file mode 100644 index 0000000000..9758377f99 --- /dev/null +++ b/tests/regressiontests/indexes/models.py @@ -0,0 +1,11 @@ +from django.db import models + + +class Article(models.Model): + headline = models.CharField(max_length=100) + pub_date = models.DateTimeField() + + class Meta: + index_together = [ + ["headline", "pub_date"], + ] diff --git a/tests/regressiontests/indexes/tests.py b/tests/regressiontests/indexes/tests.py new file mode 100644 index 0000000000..0dac881fa9 --- /dev/null +++ b/tests/regressiontests/indexes/tests.py @@ -0,0 +1,12 @@ +from django.core.management.color import no_style +from django.db import connections, DEFAULT_DB_ALIAS +from django.test import TestCase + +from .models import Article + + +class IndexesTests(TestCase): + def test_index_together(self): + connection = connections[DEFAULT_DB_ALIAS] + index_sql = connection.creation.sql_indexes_for_model(Article, no_style()) + self.assertEqual(len(index_sql), 1) diff --git a/tests/regressiontests/initial_sql_regress/tests.py b/tests/regressiontests/initial_sql_regress/tests.py index 03a91cb807..39d8921061 100644 --- a/tests/regressiontests/initial_sql_regress/tests.py +++ b/tests/regressiontests/initial_sql_regress/tests.py @@ -1,3 +1,6 @@ +from django.core.management.color import no_style +from django.core.management.sql import custom_sql_for_model +from django.db import connections, DEFAULT_DB_ALIAS from django.test import TestCase from .models import Simple @@ -15,10 +18,6 @@ class InitialSQLTests(TestCase): self.assertEqual(Simple.objects.count(), 0) def test_custom_sql(self): - from django.core.management.sql import custom_sql_for_model - from django.core.management.color import no_style - from django.db import connections, DEFAULT_DB_ALIAS - # Simulate the custom SQL loading by syncdb connection = connections[DEFAULT_DB_ALIAS] custom_sql = custom_sql_for_model(Simple, no_style(), connection) diff --git a/tests/regressiontests/introspection/models.py b/tests/regressiontests/introspection/models.py index 6e5beba61d..4de82e47e7 100644 --- a/tests/regressiontests/introspection/models.py +++ b/tests/regressiontests/introspection/models.py @@ -17,6 +17,7 @@ class Reporter(models.Model): def __str__(self): return "%s %s" % (self.first_name, self.last_name) + @python_2_unicode_compatible class Article(models.Model): headline = models.CharField(max_length=100) @@ -28,3 +29,6 @@ class Article(models.Model): class Meta: ordering = ('headline',) + index_together = [ + ["headline", "pub_date"], + ] diff --git a/tests/regressiontests/introspection/tests.py b/tests/regressiontests/introspection/tests.py index 4b8a3277e2..2df946d874 100644 --- a/tests/regressiontests/introspection/tests.py +++ b/tests/regressiontests/introspection/tests.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import,unicode_literals +from __future__ import absolute_import, unicode_literals from functools import update_wrapper @@ -13,7 +13,7 @@ if connection.vendor == 'oracle': else: expectedFailureOnOracle = lambda f: f -# + # The introspection module is optional, so methods tested here might raise # NotImplementedError. This is perfectly acceptable behavior for the backend # in question, but the tests need to handle this without failing. Ideally we'd @@ -23,7 +23,7 @@ else: # wrapper that ignores the exception. # # The metaclass is just for fun. -# + def ignore_not_implemented(func): def _inner(*args, **kwargs): @@ -34,15 +34,16 @@ def ignore_not_implemented(func): update_wrapper(_inner, func) return _inner + class IgnoreNotimplementedError(type): def __new__(cls, name, bases, attrs): - for k,v in attrs.items(): + for k, v in attrs.items(): if k.startswith('test'): attrs[k] = ignore_not_implemented(v) return type.__new__(cls, name, bases, attrs) -class IntrospectionTests(six.with_metaclass(IgnoreNotimplementedError, TestCase)): +class IntrospectionTests(six.with_metaclass(IgnoreNotimplementedError, TestCase)): def test_table_names(self): tl = connection.introspection.table_names() self.assertEqual(tl, sorted(tl)) @@ -163,6 +164,7 @@ class IntrospectionTests(six.with_metaclass(IgnoreNotimplementedError, TestCase) self.assertNotIn('first_name', indexes) self.assertIn('id', indexes) + def datatype(dbtype, description): """Helper to convert a data type into a string.""" dt = connection.introspection.get_field_type(dbtype, description) diff --git a/tests/runtests.py b/tests/runtests.py index a81fee6858..90e2dc2d65 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -277,7 +277,7 @@ if __name__ == "__main__": usage = "%prog [options] [module module module ...]" parser = OptionParser(usage=usage) parser.add_option( - '-v','--verbosity', action='store', dest='verbosity', default='1', + '-v', '--verbosity', action='store', dest='verbosity', default='1', type='choice', choices=['0', '1', '2', '3'], help='Verbosity level; 0=minimal output, 1=normal output, 2=all ' 'output') From d94dc2d1fa190d202d19a351f00a6b42e670fecb Mon Sep 17 00:00:00 2001 From: Eric Davis Date: Sun, 4 Nov 2012 09:52:37 -0800 Subject: [PATCH 078/302] Fixed formatting of get_FOO_display example --- docs/ref/models/instances.txt | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index 1ba41148b0..b4872e3e5c 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -616,25 +616,25 @@ the field. This method returns the "human-readable" value of the field. For example:: - from django.db import models + from django.db import models - class Person(models.Model): - SHIRT_SIZES = ( - (u'S', u'Small'), - (u'M', u'Medium'), - (u'L', u'Large'), - ) - name = models.CharField(max_length=60) - shirt_size = models.CharField(max_length=2, choices=SHIRT_SIZES) + class Person(models.Model): + SHIRT_SIZES = ( + (u'S', u'Small'), + (u'M', u'Medium'), + (u'L', u'Large'), + ) + name = models.CharField(max_length=60) + shirt_size = models.CharField(max_length=2, choices=SHIRT_SIZES) - :: +:: - >>> p = Person(name="Fred Flintstone", shirt_size="L") - >>> p.save() - >>> p.shirt_size - u'L' - >>> p.get_shirt_size_display() - u'Large' + >>> p = Person(name="Fred Flintstone", shirt_size="L") + >>> p.save() + >>> p.shirt_size + u'L' + >>> p.get_shirt_size_display() + u'Large' .. method:: Model.get_next_by_FOO(\**kwargs) .. method:: Model.get_previous_by_FOO(\**kwargs) From aee9c7b094cbd36c898ea465269935f156412ab9 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sun, 4 Nov 2012 12:25:48 -0800 Subject: [PATCH 079/302] Added a note and link to CLA from contributing docs --- .../contributing/writing-code/submitting-patches.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/internals/contributing/writing-code/submitting-patches.txt b/docs/internals/contributing/writing-code/submitting-patches.txt index b51ac0d906..a90dc32605 100644 --- a/docs/internals/contributing/writing-code/submitting-patches.txt +++ b/docs/internals/contributing/writing-code/submitting-patches.txt @@ -52,8 +52,15 @@ and time availability), claim it by following these steps: page, 2. then click "Submit changes." +.. note:: + The Django software foundation requests that anyone contributing more than + a trivial patch to Django sign and submit a `Contributor License + Agreement`_, this ensures that the Django Software Foundation has clear + license to all contributions allowing for a clear license for all users. + .. _Create an account: https://www.djangoproject.com/accounts/register/ .. _password reset page: https://www.djangoproject.com/accounts/password/reset/ +.. _Contributor License Agreement: https://www.djangoproject.com/foundation/cla/ Ticket claimers' responsibility ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From fcd3d4c68f06748c754c08eada526b678741e8c5 Mon Sep 17 00:00:00 2001 From: Mike Johnson Date: Sun, 4 Nov 2012 13:43:54 -0800 Subject: [PATCH 080/302] model_split: Fixed #19236 - fixed error for abstract models with a split method --- django/db/models/fields/related.py | 18 ++++++++++-------- tests/regressiontests/m2m_regress/models.py | 17 +++++++++++++++++ tests/regressiontests/m2m_regress/tests.py | 8 +++++++- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 5c3f538018..5d82c5cbb1 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -54,14 +54,16 @@ def add_lazy_relation(cls, field, relation, operation): else: # Look for an "app.Model" relation - try: - app_label, model_name = relation.split(".") - except ValueError: - # If we can't split, assume a model in current app - app_label = cls._meta.app_label - model_name = relation - except AttributeError: - # If it doesn't have a split it's actually a model class + + if isinstance(relation, basestring): + try: + app_label, model_name = relation.split(".") + except ValueError: + # If we can't split, assume a model in current app + app_label = cls._meta.app_label + model_name = relation + else: + # it's actually a model class app_label = relation._meta.app_label model_name = relation._meta.object_name diff --git a/tests/regressiontests/m2m_regress/models.py b/tests/regressiontests/m2m_regress/models.py index 7c1108456e..1a4a6df354 100644 --- a/tests/regressiontests/m2m_regress/models.py +++ b/tests/regressiontests/m2m_regress/models.py @@ -61,3 +61,20 @@ class Worksheet(models.Model): class User(models.Model): name = models.CharField(max_length=30) friends = models.ManyToManyField(auth.User) + + +class BadModelWithSplit(models.Model): + name = models.CharField(max_length=1) + + def split(self): + raise RuntimeError('split should not be called') + + class Meta: + abstract = True + + +class RegressionModelSplit(BadModelWithSplit): + """ + Model with a split method should not cause an error in add_lazy_relation + """ + others = models.ManyToManyField('self') diff --git a/tests/regressiontests/m2m_regress/tests.py b/tests/regressiontests/m2m_regress/tests.py index 92628c1825..c39d883de4 100644 --- a/tests/regressiontests/m2m_regress/tests.py +++ b/tests/regressiontests/m2m_regress/tests.py @@ -5,7 +5,7 @@ from django.test import TestCase from django.utils import six from .models import (SelfRefer, Tag, TagCollection, Entry, SelfReferChild, - SelfReferChildSibling, Worksheet) + SelfReferChildSibling, Worksheet, RegressionModelSplit) class M2MRegressionTests(TestCase): @@ -90,3 +90,9 @@ class M2MRegressionTests(TestCase): # Get same manager for different instances self.assertTrue(e1.topics.__class__ is e2.topics.__class__) self.assertTrue(t1.entry_set.__class__ is t2.entry_set.__class__) + + def test_m2m_abstract_split(self): + # Regression for #19236 - an abstract class with a 'split' method + # causes a TypeError in add_lazy_relation + m1 = RegressionModelSplit(name='1') + m1.save() From b98083ce3dfc35a3f3732f4873761a3f78e4194f Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 4 Nov 2012 13:54:54 -0800 Subject: [PATCH 081/302] Remove some bizzare and unnecesary code. --- django/contrib/contenttypes/generic.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index 726f4aa150..2dc66d07e1 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -18,6 +18,7 @@ from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets from django.contrib.contenttypes.models import ContentType from django.utils.encoding import smart_text + class GenericForeignKey(object): """ Provides a generic relation to any object through content-type/object-id @@ -51,9 +52,6 @@ class GenericForeignKey(object): kwargs[self.fk_field] = value._get_pk_val() def get_content_type(self, obj=None, id=None, using=None): - # Convenience function using get_model avoids a circular import when - # using this model - ContentType = get_model("contenttypes", "contenttype") if obj: return ContentType.objects.db_manager(obj._state.db).get_for_model(obj) elif id: @@ -215,7 +213,6 @@ class GenericRelation(RelatedField, Field): """ if negate: return [] - ContentType = get_model("contenttypes", "contenttype") content_type = ContentType.objects.get_for_model(self.model) prefix = "__".join(pieces[:pos + 1]) return [("%s__%s" % (prefix, self.content_type_field_name), From 088f68252d1f3c9e5d51a5ea3ab769397a65859f Mon Sep 17 00:00:00 2001 From: Mike Johnson Date: Sun, 4 Nov 2012 14:16:32 -0800 Subject: [PATCH 082/302] use six.string_types for python3 --- django/db/models/fields/related.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/db/models/fields/related.py b/django/db/models/fields/related.py index 5d82c5cbb1..90fe69e23c 100644 --- a/django/db/models/fields/related.py +++ b/django/db/models/fields/related.py @@ -55,7 +55,7 @@ def add_lazy_relation(cls, field, relation, operation): else: # Look for an "app.Model" relation - if isinstance(relation, basestring): + if isinstance(relation, six.string_types): try: app_label, model_name = relation.split(".") except ValueError: From 957787ace0a14fa2ee2539d47a64b266bc93b6bd Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 4 Nov 2012 15:41:33 -0800 Subject: [PATCH 083/302] Added multi-column indexes to the 1.5 release notes. --- docs/ref/models/options.txt | 4 ++-- docs/releases/1.5.txt | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/ref/models/options.txt b/docs/ref/models/options.txt index ab944d7dda..a577135271 100644 --- a/docs/ref/models/options.txt +++ b/docs/ref/models/options.txt @@ -263,10 +263,10 @@ Django quotes column and table names behind the scenes. ``index_together`` -.. versionadded:: 1.5 - .. attribute:: Options.index_together + .. versionadded:: 1.5 + Sets of field names that, taken together, are indexed:: index_together = [ diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index 154f711560..a8024424bd 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -302,6 +302,10 @@ Django 1.5 also includes several smaller improvements worth noting: * The :ref:`cache-based session backend ` can store session data in a non-default cache. +* Multi-column indexes can now be created on models. Read the + :attr:`~django.db.models.Options.index_together` documentation for more + infomration. + Backwards incompatible changes in 1.5 ===================================== From 0a49e6164c78ab6c828c08896856a77e2b423c91 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 4 Nov 2012 15:44:20 -0800 Subject: [PATCH 084/302] Corrected a typo that inadvertently made its way into the docs. --- docs/releases/1.5.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index a8024424bd..8b7af2cf36 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -304,7 +304,7 @@ Django 1.5 also includes several smaller improvements worth noting: * Multi-column indexes can now be created on models. Read the :attr:`~django.db.models.Options.index_together` documentation for more - infomration. + information. Backwards incompatible changes in 1.5 ===================================== From 5a00a57aa591c766f5ee1d8c59b64618d74fe191 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sun, 4 Nov 2012 15:46:30 -0800 Subject: [PATCH 085/302] Fixed #19240 -- include pagination error details in ListView 404 Thanks to seawolf for the patch --- django/views/generic/list.py | 7 ++++--- tests/regressiontests/generic_views/list.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/django/views/generic/list.py b/django/views/generic/list.py index ec30c58f29..946dad69f5 100644 --- a/django/views/generic/list.py +++ b/django/views/generic/list.py @@ -50,9 +50,10 @@ class MultipleObjectMixin(ContextMixin): try: page = paginator.page(page_number) return (paginator, page, page.object_list, page.has_other_pages()) - except InvalidPage: - raise Http404(_('Invalid page (%(page_number)s)') % { - 'page_number': page_number + except InvalidPage as e: + raise Http404(_('Invalid page (%(page_number)s): %(message)s') % { + 'page_number': page_number, + 'message': e.message, }) def get_paginate_by(self, queryset): diff --git a/tests/regressiontests/generic_views/list.py b/tests/regressiontests/generic_views/list.py index 14dc1d725d..6c73138043 100644 --- a/tests/regressiontests/generic_views/list.py +++ b/tests/regressiontests/generic_views/list.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from django.core.exceptions import ImproperlyConfigured from django.test import TestCase +from django.test.utils import override_settings from django.views.generic.base import View from .models import Author, Artist @@ -171,8 +172,17 @@ class ListViewTests(TestCase): with self.assertNumQueries(3): self.client.get('/list/authors/notempty/paginated/') + @override_settings(DEBUG=True) + def test_paginated_list_view_returns_useful_message_on_invalid_page(self): + # test for #19240 + # tests that source exception's message is included in page + self._make_authors(1) + res = self.client.get('/list/authors/paginated/2/') + self.assertEqual(res.status_code, 404) + self.assertEqual(res.context.get('reason'), + "Invalid page (2): That page contains no results") + def _make_authors(self, n): Author.objects.all().delete() for i in range(n): Author.objects.create(name='Author %02i' % i, slug='a%s' % i) - From e44ab5bb4fd3aa826ca4243a8ea9fd7125800da2 Mon Sep 17 00:00:00 2001 From: "Anton I. Sipos" Date: Sun, 4 Nov 2012 15:42:17 -0800 Subject: [PATCH 086/302] Fixed #18949 -- Improve performance of model_to_dict with many-to-many When calling model_to_dict, improve performance of the generated SQL by using values_list to determine primary keys of many to many objects. Add a specific test for this function, test_model_to_dict_many_to_many Thanks to brian for the original report and suggested fix. --- django/forms/models.py | 2 +- tests/modeltests/model_forms/tests.py | 36 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/django/forms/models.py b/django/forms/models.py index 11fe0c09ea..e9b71ccf26 100644 --- a/django/forms/models.py +++ b/django/forms/models.py @@ -126,7 +126,7 @@ def model_to_dict(instance, fields=None, exclude=None): data[f.name] = [] else: # MultipleChoiceWidget needs a list of pks, not object instances. - data[f.name] = [obj.pk for obj in f.value_from_object(instance)] + data[f.name] = list(f.value_from_object(instance).values_list('pk', flat=True)) else: data[f.name] = f.value_from_object(instance) return data diff --git a/tests/modeltests/model_forms/tests.py b/tests/modeltests/model_forms/tests.py index 947d0cf3c3..46d5af565a 100644 --- a/tests/modeltests/model_forms/tests.py +++ b/tests/modeltests/model_forms/tests.py @@ -561,6 +561,42 @@ class UniqueTest(TestCase): "slug": "Django 1.0"}, instance=p) self.assertTrue(form.is_valid()) +class ModelToDictTests(TestCase): + """ + Tests for forms.models.model_to_dict + """ + def test_model_to_dict_many_to_many(self): + categories=[ + Category(name='TestName1', slug='TestName1', url='url1'), + Category(name='TestName2', slug='TestName2', url='url2'), + Category(name='TestName3', slug='TestName3', url='url3') + ] + for c in categories: + c.save() + writer = Writer(name='Test writer') + writer.save() + + art = Article( + headline='Test article', + slug='test-article', + pub_date=datetime.date(1988, 1, 4), + writer=writer, + article='Hello.' + ) + art.save() + for c in categories: + art.categories.add(c) + art.save() + + with self.assertNumQueries(1): + d = model_to_dict(art) + + #Ensure all many-to-many categories appear in model_to_dict + for c in categories: + self.assertIn(c.pk, d['categories']) + #Ensure many-to-many relation appears as a list + self.assertIsInstance(d['categories'], list) + class OldFormForXTests(TestCase): def test_base_form(self): self.assertEqual(Category.objects.count(), 0) From d5c3c45f2fdfee09d81ad8dc7b0db8338d6d0aae Mon Sep 17 00:00:00 2001 From: Daniel Greenfeld Date: Sun, 4 Nov 2012 16:35:40 -0800 Subject: [PATCH 087/302] Demonstrate how to round to integers using floatformat templatetag --- docs/ref/templates/builtins.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 4aa1a990cd..175e9dfd5d 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1517,6 +1517,17 @@ displayed. For example: ``34.26000`` ``{{ value|floatformat:"-3" }}`` ``34.260`` ============ ================================ ========== +If the argument passed to ``floatformat`` is 0 (zero), it will round the number +to the nearest integer. + +============ ================================ ========== +``value`` Template Output +============ ================================ ========== +``34.23234`` ``{{ value|floatformat:"0" }}`` ``34`` +``34.00000`` ``{{ value|floatformat:"0" }}`` ``34`` +``39.56000`` ``{{ value|floatformat:"0" }}`` ``40`` +============ ================================ ========== + Using ``floatformat`` with no argument is equivalent to using ``floatformat`` with an argument of ``-1``. From a70492e6b532905c921678f35b5c60b22387f1c6 Mon Sep 17 00:00:00 2001 From: Daniel Greenfeld Date: Sun, 4 Nov 2012 16:35:40 -0800 Subject: [PATCH 088/302] Fixed #19241 -- Improved floatformat docs Demonstrate how to round to integers using floatformat templatetag --- docs/ref/templates/builtins.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 4aa1a990cd..c7fab8c53d 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1505,6 +1505,17 @@ that many decimal places. For example: ``34.26000`` ``{{ value|floatformat:3 }}`` ``34.260`` ============ ============================= ========== +Particularly useful is passing 0 (zero) as the argument which will round the +float to the nearest integer. + +============ ================================ ========== +``value`` Template Output +============ ================================ ========== +``34.23234`` ``{{ value|floatformat:"0" }}`` ``34`` +``34.00000`` ``{{ value|floatformat:"0" }}`` ``34`` +``39.56000`` ``{{ value|floatformat:"0" }}`` ``40`` +============ ================================ ========== + If the argument passed to ``floatformat`` is negative, it will round a number to that many decimal places -- but only if there's a decimal part to be displayed. For example: From fdea2621cd3f3de472afaab7aa7152a1dc4f505c Mon Sep 17 00:00:00 2001 From: "Anton I. Sipos" Date: Sun, 4 Nov 2012 18:23:03 -0800 Subject: [PATCH 089/302] Fixed #18949 -- Fix broken test interactions in ModelForms tests A test in Model Forms test was specifically referring to a fixed primary key, which was now being used up in a newly committed. This has been worked around by specifying a higher primary key. --- tests/modeltests/model_forms/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/modeltests/model_forms/tests.py b/tests/modeltests/model_forms/tests.py index 46d5af565a..c47de45ef7 100644 --- a/tests/modeltests/model_forms/tests.py +++ b/tests/modeltests/model_forms/tests.py @@ -1060,7 +1060,10 @@ class OldFormForXTests(TestCase): # Add a Category object *after* the ModelMultipleChoiceField has already been # instantiated. This proves clean() checks the database during clean() rather # than caching it at time of instantiation. - c6 = Category.objects.create(id=6, name='Sixth', url='6th') + # Note, we are using an id of 1006 here since tests that run before + # this may create categories with primary keys up to 6. Use + # a number that is will not conflict. + c6 = Category.objects.create(id=1006, name='Sixth', url='6th') self.assertEqual(c6.name, 'Sixth') self.assertQuerysetEqual(f.clean([c6.id]), ["Sixth"]) From 2cb48fffd4eecdaf77231d8acce6e6fa484698a2 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sun, 4 Nov 2012 19:04:07 -0800 Subject: [PATCH 090/302] Removed redundant docs addition across two commits d5c3c45f2fdfee09d81ad8dc7b0db8338d6d0aae a70492e6b532905c921678f35b5c60b22387f1c6 --- docs/ref/templates/builtins.txt | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index 3220b5f611..c7fab8c53d 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1528,17 +1528,6 @@ displayed. For example: ``34.26000`` ``{{ value|floatformat:"-3" }}`` ``34.260`` ============ ================================ ========== -If the argument passed to ``floatformat`` is 0 (zero), it will round the number -to the nearest integer. - -============ ================================ ========== -``value`` Template Output -============ ================================ ========== -``34.23234`` ``{{ value|floatformat:"0" }}`` ``34`` -``34.00000`` ``{{ value|floatformat:"0" }}`` ``34`` -``39.56000`` ``{{ value|floatformat:"0" }}`` ``40`` -============ ================================ ========== - Using ``floatformat`` with no argument is equivalent to using ``floatformat`` with an argument of ``-1``. From 6bd61194d49219276275542662e2fbe690534555 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sun, 4 Nov 2012 23:38:41 -0800 Subject: [PATCH 091/302] Fixed py3 compatibility for 5a00a57aa591c766f5ee1d8c59b64618d74fe191 --- django/views/generic/list.py | 2 +- tests/regressiontests/generic_views/list.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/django/views/generic/list.py b/django/views/generic/list.py index 946dad69f5..e9d57a0702 100644 --- a/django/views/generic/list.py +++ b/django/views/generic/list.py @@ -53,7 +53,7 @@ class MultipleObjectMixin(ContextMixin): except InvalidPage as e: raise Http404(_('Invalid page (%(page_number)s): %(message)s') % { 'page_number': page_number, - 'message': e.message, + 'message': str(e) }) def get_paginate_by(self, queryset): diff --git a/tests/regressiontests/generic_views/list.py b/tests/regressiontests/generic_views/list.py index 6c73138043..a8ed9e856b 100644 --- a/tests/regressiontests/generic_views/list.py +++ b/tests/regressiontests/generic_views/list.py @@ -4,6 +4,7 @@ from django.core.exceptions import ImproperlyConfigured from django.test import TestCase from django.test.utils import override_settings from django.views.generic.base import View +from django.utils.encoding import force_str from .models import Author, Artist @@ -179,7 +180,7 @@ class ListViewTests(TestCase): self._make_authors(1) res = self.client.get('/list/authors/paginated/2/') self.assertEqual(res.status_code, 404) - self.assertEqual(res.context.get('reason'), + self.assertEqual(force_str(res.context.get('reason')), "Invalid page (2): That page contains no results") def _make_authors(self, n): From 39ec43b4784d49f154ba581378fd03ec6a19e48d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 5 Nov 2012 08:42:57 +0100 Subject: [PATCH 092/302] Removed unnecessary import after b98083ce3d --- django/contrib/contenttypes/generic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/django/contrib/contenttypes/generic.py b/django/contrib/contenttypes/generic.py index 2dc66d07e1..862f2787bc 100644 --- a/django/contrib/contenttypes/generic.py +++ b/django/contrib/contenttypes/generic.py @@ -11,7 +11,6 @@ from django.db import connection from django.db.models import signals from django.db import models, router, DEFAULT_DB_ALIAS from django.db.models.fields.related import RelatedField, Field, ManyToManyRel -from django.db.models.loading import get_model from django.forms import ModelForm from django.forms.models import BaseModelFormSet, modelformset_factory, save_instance from django.contrib.admin.options import InlineModelAdmin, flatten_fieldsets From 78f66691ee7974e0ca546f09573394395a68b443 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Mon, 5 Nov 2012 20:27:06 +0100 Subject: [PATCH 093/302] Fixed #8627 -- Prevented textareas to swallow first newline content Browsers consider the first newline in textareas as some display artifact, not real content. Hence they are not sending it back to the server. If we want to keep initial newlines, we have to add one when we render the textarea. Thanks bastih for the report and initial patch. --- django/forms/widgets.py | 2 +- tests/regressiontests/forms/models.py | 6 +++++- .../forms/templates/forms/article_form.html | 8 ++++++++ tests/regressiontests/forms/tests/__init__.py | 2 +- tests/regressiontests/forms/tests/widgets.py | 20 +++++++++++++++++++ tests/regressiontests/forms/urls.py | 9 +++++++++ tests/regressiontests/forms/views.py | 8 ++++++++ 7 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/regressiontests/forms/templates/forms/article_form.html create mode 100644 tests/regressiontests/forms/urls.py create mode 100644 tests/regressiontests/forms/views.py diff --git a/django/forms/widgets.py b/django/forms/widgets.py index 061988c6c0..c761ea857d 100644 --- a/django/forms/widgets.py +++ b/django/forms/widgets.py @@ -403,7 +403,7 @@ class Textarea(Widget): def render(self, name, value, attrs=None): if value is None: value = '' final_attrs = self.build_attrs(attrs, name=name) - return format_html('{1}', + return format_html('\r\n{1}', flatatt(final_attrs), force_text(value)) diff --git a/tests/regressiontests/forms/models.py b/tests/regressiontests/forms/models.py index 6e9c269356..bec31d12d7 100644 --- a/tests/regressiontests/forms/models.py +++ b/tests/regressiontests/forms/models.py @@ -83,4 +83,8 @@ class Group(models.Model): class Cheese(models.Model): - name = models.CharField(max_length=100) + name = models.CharField(max_length=100) + + +class Article(models.Model): + content = models.TextField() diff --git a/tests/regressiontests/forms/templates/forms/article_form.html b/tests/regressiontests/forms/templates/forms/article_form.html new file mode 100644 index 0000000000..cde85051f6 --- /dev/null +++ b/tests/regressiontests/forms/templates/forms/article_form.html @@ -0,0 +1,8 @@ + + +
+ {{ form.as_p }}
+ +
+ + diff --git a/tests/regressiontests/forms/tests/__init__.py b/tests/regressiontests/forms/tests/__init__.py index 8e2150cea3..6708e54c79 100644 --- a/tests/regressiontests/forms/tests/__init__.py +++ b/tests/regressiontests/forms/tests/__init__.py @@ -18,4 +18,4 @@ from .regressions import FormsRegressionsTestCase from .util import FormsUtilTestCase from .validators import TestFieldWithValidators from .widgets import (FormsWidgetTestCase, FormsI18NWidgetsTestCase, - WidgetTests, ClearableFileInputTests) + WidgetTests, LiveWidgetTests, ClearableFileInputTests) diff --git a/tests/regressiontests/forms/tests/widgets.py b/tests/regressiontests/forms/tests/widgets.py index 88469a79e8..31c0e65c7c 100644 --- a/tests/regressiontests/forms/tests/widgets.py +++ b/tests/regressiontests/forms/tests/widgets.py @@ -4,7 +4,9 @@ from __future__ import unicode_literals import copy import datetime +from django.contrib.admin.tests import AdminSeleniumWebDriverTestCase from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.urlresolvers import reverse from django.forms import * from django.forms.widgets import RadioFieldRenderer from django.utils import formats @@ -15,6 +17,8 @@ from django.test import TestCase from django.test.utils import override_settings from django.utils.encoding import python_2_unicode_compatible +from ..models import Article + class FormsWidgetTestCase(TestCase): # Each Widget class corresponds to an HTML form widget. A Widget knows how to @@ -1095,6 +1099,22 @@ class WidgetTests(TestCase): self.assertFalse(form.is_valid()) +class LiveWidgetTests(AdminSeleniumWebDriverTestCase): + urls = 'regressiontests.forms.urls' + + def test_textarea_trailing_newlines(self): + """ + Test that a roundtrip on a ModelForm doesn't alter the TextField value + """ + article = Article.objects.create(content="\nTst\n") + self.selenium.get('%s%s' % (self.live_server_url, + reverse('article_form', args=[article.pk]))) + self.selenium.find_element_by_id('submit').submit() + article = Article.objects.get(pk=article.pk) + # Should be "\nTst\n" after #19251 is fixed + self.assertEqual(article.content, "\r\nTst\r\n") + + @python_2_unicode_compatible class FakeFieldFile(object): """ diff --git a/tests/regressiontests/forms/urls.py b/tests/regressiontests/forms/urls.py new file mode 100644 index 0000000000..b482b6c45c --- /dev/null +++ b/tests/regressiontests/forms/urls.py @@ -0,0 +1,9 @@ +from django.conf.urls import patterns, url +from django.views.generic.edit import UpdateView + +from .views import ArticleFormView + + +urlpatterns = patterns('', + url(r'^/model_form/(?P\d+)/$', ArticleFormView.as_view(), name="article_form"), +) diff --git a/tests/regressiontests/forms/views.py b/tests/regressiontests/forms/views.py new file mode 100644 index 0000000000..4bf384363c --- /dev/null +++ b/tests/regressiontests/forms/views.py @@ -0,0 +1,8 @@ +from django.views.generic.edit import UpdateView + +from .models import Article + + +class ArticleFormView(UpdateView): + model = Article + success_url = '/' From d3fd8a151231726adacb99bdbcd573f95ce32262 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Mon, 5 Nov 2012 07:14:40 -0500 Subject: [PATCH 094/302] Fixed #15591 - Clarified interaction between ModelForm and model validation. --- docs/ref/models/instances.txt | 17 ++++++++++------- docs/topics/forms/modelforms.txt | 12 +++++++++--- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/ref/models/instances.txt b/docs/ref/models/instances.txt index b4872e3e5c..1d0ac08061 100644 --- a/docs/ref/models/instances.txt +++ b/docs/ref/models/instances.txt @@ -67,9 +67,9 @@ Validating objects There are three steps involved in validating a model: -1. Validate the model fields -2. Validate the model as a whole -3. Validate the field uniqueness +1. Validate the model fields - :meth:`Model.clean_fields()` +2. Validate the model as a whole - :meth:`Model.clean()` +3. Validate the field uniqueness - :meth:`Model.validate_unique()` All three steps are performed when you call a model's :meth:`~Model.full_clean()` method. @@ -97,17 +97,20 @@ not be corrected by the user. Note that ``full_clean()`` will *not* be called automatically when you call your model's :meth:`~Model.save()` method, nor as a result of -:class:`~django.forms.ModelForm` validation. You'll need to call it manually -when you want to run one-step model validation for your own manually created -models. +:class:`~django.forms.ModelForm` validation. In the case of +:class:`~django.forms.ModelForm` validation, :meth:`Model.clean_fields()`, +:meth:`Model.clean()`, and :meth:`Model.validate_unique()` are all called +individually. -Example:: +You'll need to call ``full_clean`` manually when you want to run one-step model +validation for your own manually created models. For example:: try: article.full_clean() except ValidationError as e: # Do something based on the errors contained in e.message_dict. # Display them to a user, or handle them programatically. + pass The first step ``full_clean()`` performs is to clean each individual field. diff --git a/docs/topics/forms/modelforms.txt b/docs/topics/forms/modelforms.txt index 692be7cd7c..fcc9bd6a7f 100644 --- a/docs/topics/forms/modelforms.txt +++ b/docs/topics/forms/modelforms.txt @@ -192,6 +192,8 @@ we'll discuss in a moment.):: name = forms.CharField(max_length=100) authors = forms.ModelMultipleChoiceField(queryset=Author.objects.all()) +.. _modelform-is-valid-and-errors: + The ``is_valid()`` method and ``errors`` ---------------------------------------- @@ -213,7 +215,9 @@ method. This method creates and saves a database object from the data bound to the form. A subclass of ``ModelForm`` can accept an existing model instance as the keyword argument ``instance``; if this is supplied, ``save()`` will update that instance. If it's not supplied, -``save()`` will create a new instance of the specified model:: +``save()`` will create a new instance of the specified model: + +.. code-block:: python # Create a form instance from POST data. >>> f = ArticleForm(request.POST) @@ -232,8 +236,10 @@ supplied, ``save()`` will update that instance. If it's not supplied, >>> f = ArticleForm(request.POST, instance=a) >>> f.save() -Note that ``save()`` will raise a ``ValueError`` if the data in the form -doesn't validate -- i.e., if form.errors evaluates to True. +Note that if the form :ref:`hasn't been validated +`, calling ``save()`` will do so by checking +``form.errors``. A ``ValueError`` will be raised if the data in the form +doesn't validate -- i.e., if ``form.errors`` evaluates to ``True``. This ``save()`` method accepts an optional ``commit`` keyword argument, which accepts either ``True`` or ``False``. If you call ``save()`` with From 11fd00c46e2826ce5852e096487762a96909b717 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 6 Nov 2012 10:19:14 +0100 Subject: [PATCH 095/302] Fixed #19254 -- Bug in SESSION_FILE_PATH handling. Thanks simonb for the report. Refs #18194. --- django/contrib/sessions/backends/file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/sessions/backends/file.py b/django/contrib/sessions/backends/file.py index f3a71f8271..401c90cf74 100644 --- a/django/contrib/sessions/backends/file.py +++ b/django/contrib/sessions/backends/file.py @@ -176,7 +176,7 @@ class SessionStore(SessionBase): @classmethod def clear_expired(cls): - storage_path = getattr(settings, "SESSION_FILE_PATH", tempfile.gettempdir()) + storage_path = cls._get_storage_path() file_prefix = settings.SESSION_COOKIE_NAME for session_file in os.listdir(storage_path): From 2cc1884383a0b5371854be6806851521b623f45b Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 6 Nov 2012 05:16:01 -0500 Subject: [PATCH 096/302] Fixed #19246 - Updated SECURE_PROXY_SSL_HEADER example to use 'X-Forwarded-Proto' Thanks Fred Palmer for the report. --- docs/ref/settings.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index e8b41afb39..5544c99dd1 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1560,9 +1560,9 @@ for. You'll need to set a tuple with two elements -- the name of the header to look for and the required value. For example:: - SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') + SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -Here, we're telling Django that we trust the ``X-Forwarded-Protocol`` header +Here, we're telling Django that we trust the ``X-Forwarded-Proto`` header that comes from our proxy, and any time its value is ``'https'``, then the request is guaranteed to be secure (i.e., it originally came in via HTTPS). Obviously, you should *only* set this setting if you control your proxy or @@ -1575,16 +1575,18 @@ available in ``request.META``.) .. warning:: - **You will probably open security holes in your site if you set this without knowing what you're doing. And if you fail to set it when you should. Seriously.** + **You will probably open security holes in your site if you set this + without knowing what you're doing. And if you fail to set it when you + should. Seriously.** Make sure ALL of the following are true before setting this (assuming the values from the example above): * Your Django app is behind a proxy. - * Your proxy strips the 'X-Forwarded-Protocol' header from all incoming + * Your proxy strips the ``X-Forwarded-Proto`` header from all incoming requests. In other words, if end users include that header in their requests, the proxy will discard it. - * Your proxy sets the 'X-Forwarded-Protocol' header and sends it to Django, + * Your proxy sets the ``X-Forwarded-Proto`` header and sends it to Django, but only for requests that originally come in via HTTPS. If any of those are not true, you should keep this setting set to ``None`` From 79dd751b0b9e428bf386d7304f442fa9815fdbba Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 6 Nov 2012 12:19:14 +0100 Subject: [PATCH 097/302] Fixed #14315 -- Made memcached backend handle negative incr/decr values Thanks Michael Manfre for the report and initial patch and Tobias McNulty for the review. --- django/core/cache/backends/memcached.py | 6 ++++++ tests/regressiontests/cache/tests.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/django/core/cache/backends/memcached.py b/django/core/cache/backends/memcached.py index 9bb47c8344..2c3f198847 100644 --- a/django/core/cache/backends/memcached.py +++ b/django/core/cache/backends/memcached.py @@ -90,6 +90,9 @@ class BaseMemcachedCache(BaseCache): def incr(self, key, delta=1, version=None): key = self.make_key(key, version=version) + # memcached doesn't support a negative delta + if delta < 0: + return self._cache.decr(key, -delta) try: val = self._cache.incr(key, delta) @@ -105,6 +108,9 @@ class BaseMemcachedCache(BaseCache): def decr(self, key, delta=1, version=None): key = self.make_key(key, version=version) + # memcached doesn't support a negative delta + if delta < 0: + return self._cache.incr(key, -delta) try: val = self._cache.decr(key, delta) diff --git a/tests/regressiontests/cache/tests.py b/tests/regressiontests/cache/tests.py index a6eff8950b..9960c01300 100644 --- a/tests/regressiontests/cache/tests.py +++ b/tests/regressiontests/cache/tests.py @@ -257,6 +257,7 @@ class BaseCacheTests(object): self.assertEqual(self.cache.get('answer'), 42) self.assertEqual(self.cache.incr('answer', 10), 52) self.assertEqual(self.cache.get('answer'), 52) + self.assertEqual(self.cache.incr('answer', -10), 42) self.assertRaises(ValueError, self.cache.incr, 'does_not_exist') def test_decr(self): @@ -266,6 +267,7 @@ class BaseCacheTests(object): self.assertEqual(self.cache.get('answer'), 42) self.assertEqual(self.cache.decr('answer', 10), 32) self.assertEqual(self.cache.get('answer'), 32) + self.assertEqual(self.cache.decr('answer', -10), 42) self.assertRaises(ValueError, self.cache.decr, 'does_not_exist') def test_data_types(self): From 620e0bba4969f27230d35f75bc6a1624c3fac747 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 6 Nov 2012 08:53:10 -0500 Subject: [PATCH 098/302] Fixed #19154 - Noted commit_manually requires commit/rollback for reads Thanks als for the report. --- docs/topics/db/transactions.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/db/transactions.txt b/docs/topics/db/transactions.txt index 4a52c5af35..3c0ec4f187 100644 --- a/docs/topics/db/transactions.txt +++ b/docs/topics/db/transactions.txt @@ -161,8 +161,12 @@ managers, too. transactions. It tells Django you'll be managing the transaction on your own. - If your view changes data and doesn't ``commit()`` or ``rollback()``, - Django will raise a ``TransactionManagementError`` exception. + Whether you are writing or simply reading from the database, you must + ``commit()`` or ``rollback()`` explicitly or Django will raise a + :exc:`TransactionManagementError` exception. This is required when reading + from the database because ``SELECT`` statements may call functions which + modify tables, and thus it is impossible to know if any data has been + modified. Manual transaction management looks like this:: From a386675a6a636b8e6b96277da8e9191e9dc710dc Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 6 Nov 2012 06:58:25 -0500 Subject: [PATCH 099/302] Fixed #15968 - Noted that readonly_fields are excluded from the ModelForm --- docs/ref/contrib/admin/index.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/ref/contrib/admin/index.txt b/docs/ref/contrib/admin/index.txt index ee1342b43d..29ee66bccc 100644 --- a/docs/ref/contrib/admin/index.txt +++ b/docs/ref/contrib/admin/index.txt @@ -816,9 +816,11 @@ subclass:: By default the admin shows all fields as editable. Any fields in this option (which should be a ``list`` or ``tuple``) will display its data - as-is and non-editable. Note that when specifying :attr:`ModelAdmin.fields` - or :attr:`ModelAdmin.fieldsets` the read-only fields must be present to be - shown (they are ignored otherwise). + as-is and non-editable; they are also excluded from the + :class:`~django.forms.ModelForm` used for creating and editing. Note that + when specifying :attr:`ModelAdmin.fields` or :attr:`ModelAdmin.fieldsets` + the read-only fields must be present to be shown (they are ignored + otherwise). If ``readonly_fields`` is used without defining explicit ordering through :attr:`ModelAdmin.fields` or :attr:`ModelAdmin.fieldsets` they will be From e8f696097b19abbd3a98990ea4ca5f58d0444826 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 6 Nov 2012 19:44:49 -0500 Subject: [PATCH 100/302] Fixed #19161 - Added missing clean_password method in custom user docs Thanks DavidW for the report. --- docs/topics/auth.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/topics/auth.txt b/docs/topics/auth.txt index f7a9084008..aed482f710 100644 --- a/docs/topics/auth.txt +++ b/docs/topics/auth.txt @@ -408,7 +408,7 @@ installation supports. The first entry in this list (that is, ``settings.PASSWORD_HASHERS[0]``) will be used to store passwords, and all the other entries are valid hashers that can be used to check existing passwords. This means that if you want to use a different algorithm, you'll need to modify -:setting:`PASSWORD_HASHERS` to list your prefered algorithm first in the list. +:setting:`PASSWORD_HASHERS` to list your preferred algorithm first in the list. The default for :setting:`PASSWORD_HASHERS` is:: @@ -2283,13 +2283,19 @@ code would be required in the app's ``admin.py`` file:: class UserChangeForm(forms.ModelForm): """A form for updateing users. Includes all the fields on the user, but replaces the password field with admin's - pasword hash display field. + password hash display field. """ password = ReadOnlyPasswordHashField() class Meta: model = MyUser + def clean_password(self): + # Regardless of what the user provides, return the initial value. + # This is done here, rather than on the field, because the + # field does not have access to the initial value + return self.initial["password"] + class MyUserAdmin(UserAdmin): # The forms to add and change user instances From b0c72d0a308bcb41c676d36c24e6b2f85e00cd95 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 7 Nov 2012 17:13:06 +0100 Subject: [PATCH 101/302] Fixed invalid ipv4 mapped ipv6 addresses in docs --- docs/ref/forms/fields.txt | 2 +- docs/ref/models/fields.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ref/forms/fields.txt b/docs/ref/forms/fields.txt index 7c8d509031..75d05c6829 100644 --- a/docs/ref/forms/fields.txt +++ b/docs/ref/forms/fields.txt @@ -683,7 +683,7 @@ For each field, we describe the default widget used if you don't specify .. attribute:: unpack_ipv4 - Unpacks IPv4 mapped addresses like ``::ffff::192.0.2.1``. + Unpacks IPv4 mapped addresses like ``::ffff:192.0.2.1``. If this option is enabled that address would be unpacked to ``192.0.2.1``. Default is disabled. Can only be used when ``protocol`` is set to ``'both'``. diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt index d8ea6bb31d..cd1185585c 100644 --- a/docs/ref/models/fields.txt +++ b/docs/ref/models/fields.txt @@ -825,7 +825,7 @@ are converted to lowercase. .. attribute:: GenericIPAddressField.unpack_ipv4 - Unpacks IPv4 mapped addresses like ``::ffff::192.0.2.1``. + Unpacks IPv4 mapped addresses like ``::ffff:192.0.2.1``. If this option is enabled that address would be unpacked to ``192.0.2.1``. Default is disabled. Can only be used when ``protocol`` is set to ``'both'``. From 9a09558e9f20e088b4526fff6374a53e877cf5ec Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 7 Nov 2012 18:24:49 +0100 Subject: [PATCH 102/302] Fixed #19257 -- Don't swallow command's KeyError in call_command Thanks Giovanni Bajo for the report. --- django/core/management/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/django/core/management/__init__.py b/django/core/management/__init__.py index c61ab2b663..bb26c20666 100644 --- a/django/core/management/__init__.py +++ b/django/core/management/__init__.py @@ -136,14 +136,15 @@ def call_command(name, *args, **options): # Load the command object. try: app_name = get_commands()[name] - if isinstance(app_name, BaseCommand): - # If the command is already loaded, use it directly. - klass = app_name - else: - klass = load_command_class(app_name, name) except KeyError: raise CommandError("Unknown command: %r" % name) + if isinstance(app_name, BaseCommand): + # If the command is already loaded, use it directly. + klass = app_name + else: + klass = load_command_class(app_name, name) + # Grab out a list of defaults from the options. optparse does this for us # when the script runs from the command line, but since call_command can # be called programatically, we need to simulate the loading and handling From b1ac329ba9bbcba90d8ced7e16909ed169b1d16e Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 7 Nov 2012 18:31:14 +0100 Subject: [PATCH 103/302] Fixed #19115 -- Documented stdout/stderr options for call_command Thanks d1ffuz0r for helping with the patch. --- docs/ref/django-admin.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/ref/django-admin.txt b/docs/ref/django-admin.txt index e0b08450e9..a5ed37f25f 100644 --- a/docs/ref/django-admin.txt +++ b/docs/ref/django-admin.txt @@ -1469,3 +1469,12 @@ Examples:: from django.core import management management.call_command('flush', verbosity=0, interactive=False) management.call_command('loaddata', 'test_data', verbosity=0) + +Output redirection +================== + +Note that you can redirect standard output and error streams as all commands +support the ``stdout`` and ``stderr`` options. For example, you could write:: + + with open('/tmp/command_output') as f: + management.call_command('dumpdata', stdout=f) From cafb266954e21dd55ddfa90597bcf02c022bcb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 8 Nov 2012 00:04:57 +0200 Subject: [PATCH 104/302] Fixed #17144 -- MySQL again groups by PK only Thanks to Christian Oudard for the report and tests. --- django/db/models/sql/compiler.py | 66 +++++++------- .../aggregation_regress/tests.py | 91 ++++++++++++++++++- 2 files changed, 125 insertions(+), 32 deletions(-) diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index db1e1131ad..f0a1611e9c 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -103,21 +103,12 @@ class SQLCompiler(object): result.append('WHERE %s' % where) params.extend(w_params) - grouping, gb_params = self.get_grouping() + grouping, gb_params = self.get_grouping(ordering_group_by) if grouping: if distinct_fields: raise NotImplementedError( "annotate() + distinct(fields) not implemented.") - if ordering: - # If the backend can't group by PK (i.e., any database - # other than MySQL), then any fields mentioned in the - # ordering clause needs to be in the group by clause. - if not self.connection.features.allows_group_by_pk: - for col, col_params in ordering_group_by: - if col not in grouping: - grouping.append(str(col)) - gb_params.extend(col_params) - else: + if not ordering: ordering = self.connection.ops.force_no_ordering() result.append('GROUP BY %s' % ', '.join(grouping)) params.extend(gb_params) @@ -378,7 +369,7 @@ class SQLCompiler(object): else: order = asc result.append('%s %s' % (field, order)) - group_by.append((field, [])) + group_by.append((str(field), [])) continue col, order = get_order_dir(field, asc) if col in self.query.aggregate_select: @@ -538,39 +529,52 @@ class SQLCompiler(object): first = False return result, [] - def get_grouping(self): + def get_grouping(self, ordering_group_by): """ Returns a tuple representing the SQL elements in the "group by" clause. """ qn = self.quote_name_unless_alias result, params = [], [] if self.query.group_by is not None: - if (len(self.query.model._meta.fields) == len(self.query.select) and - self.connection.features.allows_group_by_pk): + select_cols = self.query.select + self.query.related_select_cols + # Just the column, not the fields. + select_cols = [s[0] for s in select_cols] + if (len(self.query.model._meta.fields) == len(self.query.select) + and self.connection.features.allows_group_by_pk): self.query.group_by = [ (self.query.model._meta.db_table, self.query.model._meta.pk.column) ] - - group_by = self.query.group_by or [] - - extra_selects = [] - for extra_select, extra_params in six.itervalues(self.query.extra_select): - extra_selects.append(extra_select) - params.extend(extra_params) - select_cols = [s.col for s in self.query.select] - related_select_cols = [s.col for s in self.query.related_select_cols] - cols = (group_by + select_cols + related_select_cols + extra_selects) + select_cols = [] seen = set() + cols = self.query.group_by + select_cols for col in cols: - if col in seen: - continue - seen.add(col) if isinstance(col, (list, tuple)): - result.append('%s.%s' % (qn(col[0]), qn(col[1]))) + sql = '%s.%s' % (qn(col[0]), qn(col[1])) elif hasattr(col, 'as_sql'): - result.append(col.as_sql(qn, self.connection)) + sql = col.as_sql(qn, self.connection) else: - result.append('(%s)' % str(col)) + sql = '(%s)' % str(col) + if sql not in seen: + result.append(sql) + seen.add(sql) + + # Still, we need to add all stuff in ordering (except if the backend can + # group by just by PK). + if ordering_group_by and not self.connection.features.allows_group_by_pk: + for order, order_params in ordering_group_by: + # Even if we have seen the same SQL string, it might have + # different params, so, we add same SQL in "has params" case. + if order not in seen or params: + result.append(order) + params.extend(order_params) + seen.add(order) + + # Unconditionally add the extra_select items. + for extra_select, extra_params in self.query.extra_select.values(): + sql = '(%s)' % str(extra_select) + result.append(sql) + params.extend(extra_params) + return result, params def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, diff --git a/tests/regressiontests/aggregation_regress/tests.py b/tests/regressiontests/aggregation_regress/tests.py index af0f421502..9b3cd41e41 100644 --- a/tests/regressiontests/aggregation_regress/tests.py +++ b/tests/regressiontests/aggregation_regress/tests.py @@ -470,7 +470,7 @@ class AggregationTests(TestCase): # Regression for #15709 - Ensure each group_by field only exists once # per query qs = Book.objects.values('publisher').annotate(max_pages=Max('pages')).order_by() - grouping, gb_params = qs.query.get_compiler(qs.db).get_grouping() + grouping, gb_params = qs.query.get_compiler(qs.db).get_grouping([]) self.assertEqual(len(grouping), 1) def test_duplicate_alias(self): @@ -889,3 +889,92 @@ class AggregationTests(TestCase): self.assertIs(qs.query.alias_map['aggregation_regress_book'].join_type, None) # Check that the query executes without problems. self.assertEqual(len(qs.exclude(publisher=-1)), 6) + + @skipUnlessDBFeature("allows_group_by_pk") + def test_aggregate_duplicate_columns(self): + # Regression test for #17144 + + results = Author.objects.annotate(num_contacts=Count('book_contact_set')) + + # There should only be one GROUP BY clause, for the `id` column. + # `name` and `age` should not be grouped on. + grouping, gb_params = results.query.get_compiler(using='default').get_grouping([]) + self.assertEqual(len(grouping), 1) + assert 'id' in grouping[0] + assert 'name' not in grouping[0] + assert 'age' not in grouping[0] + + # The query group_by property should also only show the `id`. + self.assertEqual(results.query.group_by, [('aggregation_regress_author', 'id')]) + + # Ensure that we get correct results. + self.assertEqual( + [(a.name, a.num_contacts) for a in results.order_by('name')], + [ + ('Adrian Holovaty', 1), + ('Brad Dayley', 1), + ('Jacob Kaplan-Moss', 0), + ('James Bennett', 1), + ('Jeffrey Forcier', 1), + ('Paul Bissex', 0), + ('Peter Norvig', 2), + ('Stuart Russell', 0), + ('Wesley J. Chun', 0), + ] + ) + + @skipUnlessDBFeature("allows_group_by_pk") + def test_aggregate_duplicate_columns_only(self): + # Works with only() too. + results = Author.objects.only('id', 'name').annotate(num_contacts=Count('book_contact_set')) + grouping, gb_params = results.query.get_compiler(using='default').get_grouping([]) + self.assertEqual(len(grouping), 1) + assert 'id' in grouping[0] + assert 'name' not in grouping[0] + assert 'age' not in grouping[0] + + # The query group_by property should also only show the `id`. + self.assertEqual(results.query.group_by, [('aggregation_regress_author', 'id')]) + + # Ensure that we get correct results. + self.assertEqual( + [(a.name, a.num_contacts) for a in results.order_by('name')], + [ + ('Adrian Holovaty', 1), + ('Brad Dayley', 1), + ('Jacob Kaplan-Moss', 0), + ('James Bennett', 1), + ('Jeffrey Forcier', 1), + ('Paul Bissex', 0), + ('Peter Norvig', 2), + ('Stuart Russell', 0), + ('Wesley J. Chun', 0), + ] + ) + + @skipUnlessDBFeature("allows_group_by_pk") + def test_aggregate_duplicate_columns_select_related(self): + # And select_related() + results = Book.objects.select_related('contact').annotate( + num_authors=Count('authors')) + grouping, gb_params = results.query.get_compiler(using='default').get_grouping([]) + self.assertEqual(len(grouping), 1) + assert 'id' in grouping[0] + assert 'name' not in grouping[0] + assert 'contact' not in grouping[0] + + # The query group_by property should also only show the `id`. + self.assertEqual(results.query.group_by, [('aggregation_regress_book', 'id')]) + + # Ensure that we get correct results. + self.assertEqual( + [(b.name, b.num_authors) for b in results.order_by('name')], + [ + ('Artificial Intelligence: A Modern Approach', 2), + ('Paradigms of Artificial Intelligence Programming: Case Studies in Common Lisp', 1), + ('Practical Django Projects', 1), + ('Python Web Development with Django', 3), + ('Sams Teach Yourself Django in 24 Hours', 1), + ('The Definitive Guide to Django: Web Development Done Right', 2) + ] + ) From 45802e12481e86f7b9c1f392a54b9d25671214d9 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 8 Nov 2012 11:00:26 +0100 Subject: [PATCH 105/302] Merged pagination tests It is simpler/cleaner to have all pagination tests in a single file. Refs #16122. --- tests/modeltests/pagination/tests.py | 134 ------------------ .../pagination/__init__.py | 0 .../pagination/models.py | 8 -- .../tests.py | 108 ++++++++++++-- .../pagination_regress/__init__.py | 0 .../pagination_regress/models.py | 1 - 6 files changed, 95 insertions(+), 156 deletions(-) delete mode 100644 tests/modeltests/pagination/tests.py rename tests/{modeltests => regressiontests}/pagination/__init__.py (100%) rename tests/{modeltests => regressiontests}/pagination/models.py (59%) rename tests/regressiontests/{pagination_regress => pagination}/tests.py (72%) delete mode 100644 tests/regressiontests/pagination_regress/__init__.py delete mode 100644 tests/regressiontests/pagination_regress/models.py diff --git a/tests/modeltests/pagination/tests.py b/tests/modeltests/pagination/tests.py deleted file mode 100644 index 12ce0e3ecb..0000000000 --- a/tests/modeltests/pagination/tests.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import absolute_import, unicode_literals - -from datetime import datetime - -from django.core.paginator import Paginator, InvalidPage, EmptyPage -from django.test import TestCase -from django.utils import six - -from .models import Article - - -class CountContainer(object): - def count(self): - return 42 - -class LenContainer(object): - def __len__(self): - return 42 - -class PaginationTests(TestCase): - def setUp(self): - # Prepare a list of objects for pagination. - for x in range(1, 10): - a = Article(headline='Article %s' % x, pub_date=datetime(2005, 7, 29)) - a.save() - - def test_paginator(self): - paginator = Paginator(Article.objects.all(), 5) - self.assertEqual(9, paginator.count) - self.assertEqual(2, paginator.num_pages) - self.assertEqual([1, 2], list(paginator.page_range)) - - def test_first_page(self): - paginator = Paginator(Article.objects.all(), 5) - p = paginator.page(1) - self.assertEqual("", six.text_type(p)) - self.assertQuerysetEqual(p.object_list, [ - "", - "", - "", - "", - "" - ] - ) - self.assertTrue(p.has_next()) - self.assertFalse(p.has_previous()) - self.assertTrue(p.has_other_pages()) - self.assertEqual(2, p.next_page_number()) - self.assertRaises(InvalidPage, p.previous_page_number) - self.assertEqual(1, p.start_index()) - self.assertEqual(5, p.end_index()) - - def test_last_page(self): - paginator = Paginator(Article.objects.all(), 5) - p = paginator.page(2) - self.assertEqual("", six.text_type(p)) - self.assertQuerysetEqual(p.object_list, [ - "", - "", - "", - "" - ] - ) - self.assertFalse(p.has_next()) - self.assertTrue(p.has_previous()) - self.assertTrue(p.has_other_pages()) - self.assertRaises(InvalidPage, p.next_page_number) - self.assertEqual(1, p.previous_page_number()) - self.assertEqual(6, p.start_index()) - self.assertEqual(9, p.end_index()) - - def test_empty_page(self): - paginator = Paginator(Article.objects.all(), 5) - self.assertRaises(EmptyPage, paginator.page, 0) - self.assertRaises(EmptyPage, paginator.page, 3) - - # Empty paginators with allow_empty_first_page=True. - paginator = Paginator(Article.objects.filter(id=0), 5, allow_empty_first_page=True) - self.assertEqual(0, paginator.count) - self.assertEqual(1, paginator.num_pages) - self.assertEqual([1], list(paginator.page_range)) - - # Empty paginators with allow_empty_first_page=False. - paginator = Paginator(Article.objects.filter(id=0), 5, allow_empty_first_page=False) - self.assertEqual(0, paginator.count) - self.assertEqual(0, paginator.num_pages) - self.assertEqual([], list(paginator.page_range)) - - def test_invalid_page(self): - paginator = Paginator(Article.objects.all(), 5) - self.assertRaises(InvalidPage, paginator.page, 7) - - def test_orphans(self): - # Add a few more records to test out the orphans feature. - for x in range(10, 13): - Article(headline="Article %s" % x, pub_date=datetime(2006, 10, 6)).save() - - # With orphans set to 3 and 10 items per page, we should get all 12 items on a single page. - paginator = Paginator(Article.objects.all(), 10, orphans=3) - self.assertEqual(1, paginator.num_pages) - - # With orphans only set to 1, we should get two pages. - paginator = Paginator(Article.objects.all(), 10, orphans=1) - self.assertEqual(2, paginator.num_pages) - - def test_paginate_list(self): - # Paginators work with regular lists/tuples, too -- not just with QuerySets. - paginator = Paginator([1, 2, 3, 4, 5, 6, 7, 8, 9], 3) - self.assertEqual(9, paginator.count) - self.assertEqual(3, paginator.num_pages) - self.assertEqual([1, 2, 3], list(paginator.page_range)) - p = paginator.page(2) - self.assertEqual("", six.text_type(p)) - self.assertEqual([4, 5, 6], p.object_list) - self.assertTrue(p.has_next()) - self.assertTrue(p.has_previous()) - self.assertTrue(p.has_other_pages()) - self.assertEqual(3, p.next_page_number()) - self.assertEqual(1, p.previous_page_number()) - self.assertEqual(4, p.start_index()) - self.assertEqual(6, p.end_index()) - - def test_paginate_misc_classes(self): - # Paginator can be passed other objects with a count() method. - paginator = Paginator(CountContainer(), 10) - self.assertEqual(42, paginator.count) - self.assertEqual(5, paginator.num_pages) - self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range)) - - # Paginator can be passed other objects that implement __len__. - paginator = Paginator(LenContainer(), 10) - self.assertEqual(42, paginator.count) - self.assertEqual(5, paginator.num_pages) - self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range)) diff --git a/tests/modeltests/pagination/__init__.py b/tests/regressiontests/pagination/__init__.py similarity index 100% rename from tests/modeltests/pagination/__init__.py rename to tests/regressiontests/pagination/__init__.py diff --git a/tests/modeltests/pagination/models.py b/tests/regressiontests/pagination/models.py similarity index 59% rename from tests/modeltests/pagination/models.py rename to tests/regressiontests/pagination/models.py index 779d3029ba..9dc8d4b776 100644 --- a/tests/modeltests/pagination/models.py +++ b/tests/regressiontests/pagination/models.py @@ -1,11 +1,3 @@ -""" -30. Object pagination - -Django provides a framework for paginating a list of objects in a few lines -of code. This is often useful for dividing search results or long lists of -objects into easily readable pages. -""" - from django.db import models from django.utils.encoding import python_2_unicode_compatible diff --git a/tests/regressiontests/pagination_regress/tests.py b/tests/regressiontests/pagination/tests.py similarity index 72% rename from tests/regressiontests/pagination_regress/tests.py rename to tests/regressiontests/pagination/tests.py index e98352e006..a49f9b8fa1 100644 --- a/tests/regressiontests/pagination_regress/tests.py +++ b/tests/regressiontests/pagination/tests.py @@ -1,9 +1,17 @@ -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals -from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger -from django.utils.unittest import TestCase +from datetime import datetime -class PaginatorTests(TestCase): +from django.core.paginator import (Paginator, EmptyPage, InvalidPage, + PageNotAnInteger) +from django.test import TestCase +from django.utils import six +from django.utils import unittest + +from .models import Article + + +class PaginationTests(unittest.TestCase): """ Tests for the Paginator and Page classes. """ @@ -31,15 +39,6 @@ class PaginatorTests(TestCase): "For '%s', expected %s but got %s. Paginator parameters were: %s" % (name, expected, got, params)) - def test_invalid_page_number(self): - """ - Tests that invalid page numbers result in the correct exception being - raised. - """ - paginator = Paginator([1, 2, 3], 2) - self.assertRaises(PageNotAnInteger, paginator.validate_number, None) - self.assertRaises(PageNotAnInteger, paginator.validate_number, 'x') - def test_paginator(self): """ Tests the paginator attributes using varying inputs. @@ -107,6 +106,38 @@ class PaginatorTests(TestCase): for params, output in tests: self.check_paginator(params, output) + def test_invalid_page_number(self): + """ + Tests that invalid page numbers result in the correct exception being + raised. + """ + paginator = Paginator([1, 2, 3], 2) + self.assertRaises(InvalidPage, paginator.page, 3) + self.assertRaises(PageNotAnInteger, paginator.validate_number, None) + self.assertRaises(PageNotAnInteger, paginator.validate_number, 'x') + # With no content and allow_empty_first_page=True, 1 is a valid page number + paginator = Paginator([], 2) + self.assertEqual(paginator.validate_number(1), 1) + + def test_paginate_misc_classes(self): + class CountContainer(object): + def count(self): + return 42 + # Paginator can be passed other objects with a count() method. + paginator = Paginator(CountContainer(), 10) + self.assertEqual(42, paginator.count) + self.assertEqual(5, paginator.num_pages) + self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range)) + + # Paginator can be passed other objects that implement __len__. + class LenContainer(object): + def __len__(self): + return 42 + paginator = Paginator(LenContainer(), 10) + self.assertEqual(42, paginator.count) + self.assertEqual(5, paginator.num_pages) + self.assertEqual([1, 2, 3, 4, 5], list(paginator.page_range)) + def check_indexes(self, params, page_num, indexes): """ Helper method that instantiates a Paginator object from the passed @@ -168,6 +199,7 @@ class PaginatorTests(TestCase): for params, first, last in tests: self.check_indexes(params, 'first', first) self.check_indexes(params, 'last', last) + # When no items and no empty first page, we should get EmptyPage error. self.assertRaises(EmptyPage, self.check_indexes, ([], 4, 0, False), 1, None) self.assertRaises(EmptyPage, self.check_indexes, ([], 4, 1, False), 1, None) @@ -184,3 +216,53 @@ class PaginatorTests(TestCase): self.assertFalse('a' in page2) self.assertEqual(''.join(page2), 'fghijk') self.assertEqual(''.join(reversed(page2)), 'kjihgf') + + +class ModelPaginationTests(TestCase): + """ + Test pagination with Django model instances + """ + def setUp(self): + # Prepare a list of objects for pagination. + for x in range(1, 10): + a = Article(headline='Article %s' % x, pub_date=datetime(2005, 7, 29)) + a.save() + + def test_first_page(self): + paginator = Paginator(Article.objects.all(), 5) + p = paginator.page(1) + self.assertEqual("", six.text_type(p)) + self.assertQuerysetEqual(p.object_list, [ + "", + "", + "", + "", + "" + ] + ) + self.assertTrue(p.has_next()) + self.assertFalse(p.has_previous()) + self.assertTrue(p.has_other_pages()) + self.assertEqual(2, p.next_page_number()) + self.assertRaises(InvalidPage, p.previous_page_number) + self.assertEqual(1, p.start_index()) + self.assertEqual(5, p.end_index()) + + def test_last_page(self): + paginator = Paginator(Article.objects.all(), 5) + p = paginator.page(2) + self.assertEqual("", six.text_type(p)) + self.assertQuerysetEqual(p.object_list, [ + "", + "", + "", + "" + ] + ) + self.assertFalse(p.has_next()) + self.assertTrue(p.has_previous()) + self.assertTrue(p.has_other_pages()) + self.assertRaises(InvalidPage, p.next_page_number) + self.assertEqual(1, p.previous_page_number()) + self.assertEqual(6, p.start_index()) + self.assertEqual(9, p.end_index()) diff --git a/tests/regressiontests/pagination_regress/__init__.py b/tests/regressiontests/pagination_regress/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/regressiontests/pagination_regress/models.py b/tests/regressiontests/pagination_regress/models.py deleted file mode 100644 index cde172db68..0000000000 --- a/tests/regressiontests/pagination_regress/models.py +++ /dev/null @@ -1 +0,0 @@ -# Models file for tests to run. From 9942adac17f32a09cef77e4016627a6990ff7580 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 8 Nov 2012 16:13:23 +0100 Subject: [PATCH 106/302] Made Page class inherit collections.Sequence --- django/core/paginator.py | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/django/core/paginator.py b/django/core/paginator.py index 6b0b3542f8..6c502ae5f3 100644 --- a/django/core/paginator.py +++ b/django/core/paginator.py @@ -1,3 +1,4 @@ +import collections from math import ceil class InvalidPage(Exception): @@ -75,7 +76,7 @@ class Paginator(object): QuerySetPaginator = Paginator # For backwards-compatibility. -class Page(object): +class Page(collections.Sequence): def __init__(self, object_list, number, paginator): self.object_list = object_list self.number = number @@ -92,36 +93,6 @@ class Page(object): # it won't be a database hit per __getitem__. return list(self.object_list)[index] - # The following four methods are only necessary for Python <2.6 - # compatibility (this class could just extend 2.6's collections.Sequence). - - def __iter__(self): - i = 0 - try: - while True: - v = self[i] - yield v - i += 1 - except IndexError: - return - - def __contains__(self, value): - for v in self: - if v == value: - return True - return False - - def index(self, value): - for i, v in enumerate(self): - if v == value: - return i - raise ValueError - - def count(self, value): - return sum([1 for v in self if v == value]) - - # End of compatibility methods. - def has_next(self): return self.number < self.paginator.num_pages From 1db5d8827351acd2d039c218723d5100dc7e7f95 Mon Sep 17 00:00:00 2001 From: Daniel Greenfeld Date: Thu, 8 Nov 2012 16:32:16 -0800 Subject: [PATCH 107/302] Added examples for comment, templatetag, escape, force_escape, timesince, and timeuntil --- docs/ref/templates/builtins.txt | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index c7fab8c53d..ef9aa83955 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -53,6 +53,13 @@ comment Ignores everything between ``{% comment %}`` and ``{% endcomment %}``. +Sample usage:: + +

Rendered text with {{ pub_date|date:"c" }}

+ {% comment %} +

Commented out text with {{ create_date|date:"c" }}

+ {% endcomment %} + .. templatetag:: csrf_token csrf_token @@ -947,6 +954,10 @@ Argument Outputs ``closecomment`` ``#}`` ================== ======= +Sample usage:: + + {% templatetag openblock %} url 'entry_list' {% templatetag closeblock %} + .. templatetag:: url url @@ -1409,6 +1420,12 @@ applied to the result will only result in one round of escaping being done. So it is safe to use this function even in auto-escaping environments. If you want multiple escaping passes to be applied, use the :tfilter:`force_escape` filter. +For example, you can apply ``escape`` to fields when :ttag:`autoescape` is off:: + + {% autoescape off %} + {{ title|escape }} + {% endautoescape %} + .. templatefilter:: escapejs escapejs @@ -1542,6 +1559,13 @@ string. This is useful in the rare cases where you need multiple escaping or want to apply other filters to the escaped results. Normally, you want to use the :tfilter:`escape` filter. +For example, if you want to catch the ```` HTML elements created by +the :tfilter:`linebreaks` filter:: + + {% autoescape off %} + {{ body|linebreaks|force_escape }} + {% endautoescape %} + .. templatefilter:: get_digit get_digit @@ -1979,7 +2003,9 @@ Takes an optional argument that is a variable containing the date to use as the comparison point (without the argument, the comparison point is *now*). For example, if ``blog_date`` is a date instance representing midnight on 1 June 2006, and ``comment_date`` is a date instance for 08:00 on 1 June 2006, -then ``{{ blog_date|timesince:comment_date }}`` would return "8 hours". +then the following would return "8 hours":: + + {{ blog_date|timesince:comment_date }} Comparing offset-naive and offset-aware datetimes will return an empty string. @@ -1998,7 +2024,9 @@ given date or datetime. For example, if today is 1 June 2006 and Takes an optional argument that is a variable containing the date to use as the comparison point (instead of *now*). If ``from_date`` contains 22 June -2006, then ``{{ conference_date|timeuntil:from_date }}`` will return "1 week". +2006, then the following will return "1 week":: + + {{ conference_date|timeuntil:from_date }} Comparing offset-naive and offset-aware datetimes will return an empty string. From a79d920a56e7200b6259e60f7811162c07c7651d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 9 Nov 2012 09:00:27 +0100 Subject: [PATCH 108/302] Fixed #19266 -- Added Texinfo documentation target Thanks orontee for the report and initial patch. --- docs/Makefile | 6 ++++++ docs/conf.py | 10 ++++++++++ docs/make.bat | 9 +++++++++ 3 files changed, 25 insertions(+) diff --git a/docs/Makefile b/docs/Makefile index bdf48549a3..f6293a8e7f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -31,6 +31,7 @@ help: @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" + @echo " texinfo to make a Texinfo source file" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @@ -116,6 +117,11 @@ man: @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished; the Texinfo files are in $(BUILDDIR)/texinfo." + gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo diff --git a/docs/conf.py b/docs/conf.py index ced3fef5f7..939d70b7b4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -248,6 +248,16 @@ man_pages = [ ] +# -- Options for Texinfo output ------------------------------------------------ + +# List of tuples (startdocname, targetname, title, author, dir_entry, +# description, category, toctree_only) +texinfo_documents=[( + master_doc, "django", "", "", "Django", + "Documentation of the Django framework", "Web development", False +)] + + # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. diff --git a/docs/make.bat b/docs/make.bat index d6299521eb..d7f54b2059 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -29,6 +29,7 @@ if "%1" == "help" ( echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter echo. text to make text files echo. man to make manual pages + echo. texinfo to make a Texinfo source file echo. gettext to make PO message catalogs echo. changes to make an overview over all changed/added/deprecated items echo. linkcheck to check all external links for integrity @@ -143,6 +144,14 @@ if "%1" == "man" ( goto end ) +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + if "%%1" == "gettext" ( %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale if errorlevel 1 exit /b 1 From aea8bf06620c931f7b1e7d991497d593b91f71c9 Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Fri, 9 Nov 2012 15:16:06 +0100 Subject: [PATCH 109/302] Added missing encoding preamble to gis tests. 'coverage html' did fail without it. Thanks to Claude Paroz for figuring it out. --- django/contrib/gis/geoip/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django/contrib/gis/geoip/tests.py b/django/contrib/gis/geoip/tests.py index e53230d9ad..c890c4f4ba 100644 --- a/django/contrib/gis/geoip/tests.py +++ b/django/contrib/gis/geoip/tests.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import unicode_literals import os From 5bc6929f9af4d71065bce42578e49354096a7bf4 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Nov 2012 16:15:19 +0000 Subject: [PATCH 110/302] Include `versionadded 1.5` directive --- docs/ref/class-based-views/mixins-multiple-object.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ref/class-based-views/mixins-multiple-object.txt b/docs/ref/class-based-views/mixins-multiple-object.txt index e6abf26e6a..bdcb43b163 100644 --- a/docs/ref/class-based-views/mixins-multiple-object.txt +++ b/docs/ref/class-based-views/mixins-multiple-object.txt @@ -74,6 +74,8 @@ MultipleObjectMixin .. attribute:: page_kwarg + .. versionadded:: 1.5 + A string specifying the name to use for the page parameter. The view will expect this prameter to be available either as a query string parameter (via ``request.GET``) or as a kwarg variable specified From 3f2fc2f41abf226913517eb1e655f823f2c5e53a Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Fri, 9 Nov 2012 16:16:58 +0000 Subject: [PATCH 111/302] Formatting tweaks. --- docs/ref/class-based-views/mixins-multiple-object.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/class-based-views/mixins-multiple-object.txt b/docs/ref/class-based-views/mixins-multiple-object.txt index bdcb43b163..fb5a1715e6 100644 --- a/docs/ref/class-based-views/mixins-multiple-object.txt +++ b/docs/ref/class-based-views/mixins-multiple-object.txt @@ -79,7 +79,7 @@ MultipleObjectMixin A string specifying the name to use for the page parameter. The view will expect this prameter to be available either as a query string parameter (via ``request.GET``) or as a kwarg variable specified - in the URLconf. Defaults to ``"page"``. + in the URLconf. Defaults to ``page``. .. attribute:: paginator_class From 1b307d6c8fe9a897da2daa189b7d5f59120e4410 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 9 Nov 2012 19:37:50 +0100 Subject: [PATCH 112/302] Fixed #19261 -- Delayed Queryset evaluation in paginators Thanks trbs for the report and the patch. --- django/core/paginator.py | 5 +++++ tests/regressiontests/pagination/tests.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/django/core/paginator.py b/django/core/paginator.py index 6c502ae5f3..d22e16c452 100644 --- a/django/core/paginator.py +++ b/django/core/paginator.py @@ -1,6 +1,9 @@ import collections from math import ceil +from django.utils import six + + class InvalidPage(Exception): pass @@ -89,6 +92,8 @@ class Page(collections.Sequence): return len(self.object_list) def __getitem__(self, index): + if not isinstance(index, (slice,) + six.integer_types): + raise TypeError # The object_list is converted to a list so that if it was a QuerySet # it won't be a database hit per __getitem__. return list(self.object_list)[index] diff --git a/tests/regressiontests/pagination/tests.py b/tests/regressiontests/pagination/tests.py index a49f9b8fa1..63ccd8f61c 100644 --- a/tests/regressiontests/pagination/tests.py +++ b/tests/regressiontests/pagination/tests.py @@ -266,3 +266,25 @@ class ModelPaginationTests(TestCase): self.assertEqual(1, p.previous_page_number()) self.assertEqual(6, p.start_index()) self.assertEqual(9, p.end_index()) + + def test_page_getitem(self): + """ + Tests proper behaviour of a paginator page __getitem__ (queryset + evaluation, slicing, exception raised). + """ + paginator = Paginator(Article.objects.all(), 5) + p = paginator.page(1) + + # Make sure object_list queryset is not evaluated by an invalid __getitem__ call. + # (this happens from the template engine when using eg: {% page_obj.has_previous %}) + self.assertIsNone(p.object_list._result_cache) + self.assertRaises(TypeError, lambda: p['has_previous']) + self.assertIsNone(p.object_list._result_cache) + + # Make sure slicing the Page object with numbers and slice objects work. + self.assertEqual(p[0], Article.objects.get(headline='Article 1')) + self.assertQuerysetEqual(p[slice(2)], [ + "", + "", + ] + ) From 4d817b38875c900d70793acd528afc9e954bbcb7 Mon Sep 17 00:00:00 2001 From: Sean Breant Date: Fri, 9 Nov 2012 21:07:53 +0100 Subject: [PATCH 113/302] Fixed #19262 -- Support cookie pickling in SimpleTemplateResponse Refs #15863. --- django/template/response.py | 2 +- tests/regressiontests/templates/response.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/django/template/response.py b/django/template/response.py index 2cb44d127d..3b3b41331a 100644 --- a/django/template/response.py +++ b/django/template/response.py @@ -40,7 +40,7 @@ class SimpleTemplateResponse(HttpResponse): rendered, and that the pickled state only includes rendered data, not the data used to construct the response. """ - obj_dict = self.__dict__.copy() + obj_dict = super(SimpleTemplateResponse, self).__getstate__() if not self._is_rendered: raise ContentNotRenderedError('The response content must be ' 'rendered before it can be pickled.') diff --git a/tests/regressiontests/templates/response.py b/tests/regressiontests/templates/response.py index 93919b95cd..a2a76a3310 100644 --- a/tests/regressiontests/templates/response.py +++ b/tests/regressiontests/templates/response.py @@ -189,6 +189,21 @@ class SimpleTemplateResponseTest(TestCase): unpickled_response = pickle.loads(pickled_response) repickled_response = pickle.dumps(unpickled_response) + def test_pickling_cookie(self): + response = SimpleTemplateResponse('first/test.html', { + 'value': 123, + 'fn': datetime.now, + }) + + response.cookies['key'] = 'value' + + response.render() + pickled_response = pickle.dumps(response, pickle.HIGHEST_PROTOCOL) + unpickled_response = pickle.loads(pickled_response) + + self.assertEqual(unpickled_response.cookies['key'].value, 'value') + + @override_settings( TEMPLATE_CONTEXT_PROCESSORS=[test_processor_name], TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__),'templates')), From 34162698cc7a3c75b1d1b2e18b481aa7e865dc98 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 10 Nov 2012 12:05:58 +0100 Subject: [PATCH 114/302] Fixed #14264 -- Ensured settings.configure configures logging Thanks Matt McDonald for the patch. --- django/conf/__init__.py | 1 + tests/regressiontests/logging_tests/tests.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/django/conf/__init__.py b/django/conf/__init__.py index 1804c851bf..4e6f0f9211 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -83,6 +83,7 @@ class LazySettings(LazyObject): for name, value in options.items(): setattr(holder, name, value) self._wrapped = holder + self._configure_logging() @property def configured(self): diff --git a/tests/regressiontests/logging_tests/tests.py b/tests/regressiontests/logging_tests/tests.py index e40800efde..96f81981c6 100644 --- a/tests/regressiontests/logging_tests/tests.py +++ b/tests/regressiontests/logging_tests/tests.py @@ -4,7 +4,7 @@ import copy import logging import warnings -from django.conf import compat_patch_logging_config +from django.conf import compat_patch_logging_config, LazySettings from django.core import mail from django.test import TestCase, RequestFactory from django.test.utils import override_settings @@ -302,3 +302,20 @@ class SettingsConfigTest(AdminScriptTestCase): out, err = self.run_manage(['validate']) self.assertNoOutput(err) self.assertOutput(out, "0 errors found") + + +def dictConfig(config): + dictConfig.called = True +dictConfig.called = False + + +class SettingsConfigureLogging(TestCase): + """ + Test that calling settings.configure() initializes the logging + configuration. + """ + def test_configure_initializes_logging(self): + settings = LazySettings() + settings.configure( + LOGGING_CONFIG='regressiontests.logging_tests.tests.dictConfig') + self.assertTrue(dictConfig.called) From 04a7ea3283318dbec84529b055548042a43e4f4d Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 10 Nov 2012 15:42:27 +0100 Subject: [PATCH 115/302] Removed an impossible code path in cache function --- django/core/cache/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django/core/cache/__init__.py b/django/core/cache/__init__.py index f496c35e2b..d1a02a9f13 100644 --- a/django/core/cache/__init__.py +++ b/django/core/cache/__init__.py @@ -93,8 +93,6 @@ def parse_backend_conf(backend, **kwargs): raise InvalidCacheBackendError("Could not find backend '%s'" % backend) location = kwargs.pop('LOCATION', '') return backend, location, kwargs - raise InvalidCacheBackendError( - "Couldn't find a cache backend named '%s'" % backend) def get_cache(backend, **kwargs): """ From bfdedb687abfbdf400387092762bb7081e8e8b56 Mon Sep 17 00:00:00 2001 From: Ryan Kaskel Date: Sat, 10 Nov 2012 15:48:46 +0000 Subject: [PATCH 116/302] Allow custom User models to use the UserAdmin's change password view. --- django/contrib/auth/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/contrib/auth/admin.py b/django/contrib/auth/admin.py index 5f476f91c2..86442b9078 100644 --- a/django/contrib/auth/admin.py +++ b/django/contrib/auth/admin.py @@ -133,7 +133,7 @@ class UserAdmin(admin.ModelAdmin): adminForm = admin.helpers.AdminForm(form, fieldsets, {}) context = { - 'title': _('Change password: %s') % escape(user.username), + 'title': _('Change password: %s') % escape(user.get_username()), 'adminForm': adminForm, 'form_url': form_url, 'form': form, From cc0ac26f4a3947be8a3fc55d4784d3474b640c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sat, 10 Nov 2012 18:39:59 +0200 Subject: [PATCH 117/302] Fixed #19273 -- Fixed DB cache backend on pg 9.0+ and py3 There was a problem caused by Postgres 9.0+ having bytea_output default value of 'hex' and cache backend inserting the content as 'bytes' into a column of type TEXT. Fixed by converting the bytes value to a string before insert. --- django/core/cache/backends/db.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/django/core/cache/backends/db.py b/django/core/cache/backends/db.py index 348b03f733..c93bc90b18 100644 --- a/django/core/cache/backends/db.py +++ b/django/core/cache/backends/db.py @@ -11,7 +11,7 @@ except ImportError: from django.conf import settings from django.core.cache.backends.base import BaseCache from django.db import connections, router, transaction, DatabaseError -from django.utils import timezone +from django.utils import timezone, six from django.utils.encoding import force_bytes @@ -104,7 +104,11 @@ class DatabaseCache(BaseDatabaseCache): if num > self._max_entries: self._cull(db, cursor, now) pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) - encoded = base64.b64encode(pickled).strip() + b64encoded = base64.b64encode(pickled) + # The DB column is expecting a string, so make sure the value is a + # string, not bytes. Refs #19274. + if six.PY3: + b64encoded = b64encoded.decode('latin1') cursor.execute("SELECT cache_key, expires FROM %s " "WHERE cache_key = %%s" % table, [key]) try: @@ -113,11 +117,11 @@ class DatabaseCache(BaseDatabaseCache): (mode == 'add' and result[1] < now)): cursor.execute("UPDATE %s SET value = %%s, expires = %%s " "WHERE cache_key = %%s" % table, - [encoded, connections[db].ops.value_to_db_datetime(exp), key]) + [b64encoded, connections[db].ops.value_to_db_datetime(exp), key]) else: cursor.execute("INSERT INTO %s (cache_key, value, expires) " "VALUES (%%s, %%s, %%s)" % table, - [key, encoded, connections[db].ops.value_to_db_datetime(exp)]) + [key, b64encoded, connections[db].ops.value_to_db_datetime(exp)]) except DatabaseError: # To be threadsafe, updates/inserts are allowed to fail silently transaction.rollback_unless_managed(using=db) From 6c69de80bdcd2744bc64cb933c2d863dd5e74a33 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 11 Nov 2012 21:23:45 +0100 Subject: [PATCH 118/302] Tweaked cache key creation to avoid strict typing. This is a provisional change. See #19221 for details. --- django/core/cache/backends/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django/core/cache/backends/base.py b/django/core/cache/backends/base.py index 25fbbcbbdd..7234d3c4db 100644 --- a/django/core/cache/backends/base.py +++ b/django/core/cache/backends/base.py @@ -14,6 +14,7 @@ class InvalidCacheBackendError(ImproperlyConfigured): class CacheKeyWarning(DjangoRuntimeWarning): pass + # Memcached does not accept keys longer than this. MEMCACHE_MAX_KEY_LENGTH = 250 @@ -26,7 +27,7 @@ def default_key_func(key, key_prefix, version): the `key_prefix'. KEY_FUNCTION can be used to specify an alternate function with custom key making behavior. """ - return ':'.join([key_prefix, str(version), key]) + return '%s:%s:%s' % (key_prefix, version, key) def get_key_func(key_func): From 09a39ca0824611280ae180fe4cc59df669097ecb Mon Sep 17 00:00:00 2001 From: Adrian Holovaty Date: Mon, 12 Nov 2012 14:19:11 -0600 Subject: [PATCH 119/302] Negligible spacing fix in utils/log.py --- django/utils/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django/utils/log.py b/django/utils/log.py index ea0122794b..342ca1fc10 100644 --- a/django/utils/log.py +++ b/django/utils/log.py @@ -39,7 +39,7 @@ DEFAULT_LOGGING = { }, }, 'handlers': { - 'console':{ + 'console': { 'level': 'INFO', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', From 17b14d481984f3a466a02993b35940430e5dbe91 Mon Sep 17 00:00:00 2001 From: Nicolas Ippolito Date: Mon, 12 Nov 2012 22:15:41 +0100 Subject: [PATCH 120/302] Typo in comments doc --- docs/ref/contrib/comments/index.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/contrib/comments/index.txt b/docs/ref/contrib/comments/index.txt index 1c6ff7c7ed..adda1eaf6c 100644 --- a/docs/ref/contrib/comments/index.txt +++ b/docs/ref/contrib/comments/index.txt @@ -209,7 +209,7 @@ default version of which is included with Django. Rendering a custom comment form ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you want more control over the look and feel of the comment form, you use use +If you want more control over the look and feel of the comment form, you may use :ttag:`get_comment_form` to get a :doc:`form object ` that you can use in the template:: From 3f65f751a0082b83ff24849a1445aa0838b27d93 Mon Sep 17 00:00:00 2001 From: Daniel Greenfeld Date: Mon, 12 Nov 2012 16:12:27 -0800 Subject: [PATCH 121/302] Converted to

per #aaugustin's request --- docs/ref/templates/builtins.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ref/templates/builtins.txt b/docs/ref/templates/builtins.txt index ef9aa83955..5b76952c49 100644 --- a/docs/ref/templates/builtins.txt +++ b/docs/ref/templates/builtins.txt @@ -1559,7 +1559,7 @@ string. This is useful in the rare cases where you need multiple escaping or want to apply other filters to the escaped results. Normally, you want to use the :tfilter:`escape` filter. -For example, if you want to catch the ```` HTML elements created by +For example, if you want to catch the ``

`` HTML elements created by the :tfilter:`linebreaks` filter:: {% autoescape off %} From a72b8a224721775b31e2075c99165f1e3ca28092 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 13 Nov 2012 05:45:08 -0500 Subject: [PATCH 122/302] Fixed #19260 - Added a comment to tutorial 1. Thanks terwey for the suggestion. --- docs/intro/tutorial01.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/intro/tutorial01.txt b/docs/intro/tutorial01.txt index 1e2231d1e0..5712d557c6 100644 --- a/docs/intro/tutorial01.txt +++ b/docs/intro/tutorial01.txt @@ -52,7 +52,7 @@ code, then run the following command: django-admin.py startproject mysite -This will create a ``mysite`` directory in your current directory. If it didn't +This will create a ``mysite`` directory in your current directory. If it didn't work, see :doc:`Troubleshooting `. .. admonition:: Script name may differ in distribution packages @@ -666,6 +666,7 @@ Save these changes and start a new Python interactive shell by running >>> Poll.objects.get(pub_date__year=2012) + # Request an ID that doesn't exist, this will raise an exception. >>> Poll.objects.get(id=2) Traceback (most recent call last): ... From 00ff69a827b38054afe557fc7d0a589b270ed871 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Tue, 13 Nov 2012 20:46:29 +0100 Subject: [PATCH 123/302] Fixed #19283 -- Fixed typo in imports in CBV docs. --- docs/ref/class-based-views/generic-editing.txt | 2 +- docs/topics/class-based-views/generic-editing.txt | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/ref/class-based-views/generic-editing.txt b/docs/ref/class-based-views/generic-editing.txt index 2fac06ee02..01f9e32c53 100644 --- a/docs/ref/class-based-views/generic-editing.txt +++ b/docs/ref/class-based-views/generic-editing.txt @@ -16,8 +16,8 @@ editing content: has been defined. For these cases we assume the following has been defined in `myapp/models.py`:: - from django import models from django.core.urlresolvers import reverse + from django.db import models class Author(models.Model): name = models.CharField(max_length=200) diff --git a/docs/topics/class-based-views/generic-editing.txt b/docs/topics/class-based-views/generic-editing.txt index 7bae3c692d..7d12184705 100644 --- a/docs/topics/class-based-views/generic-editing.txt +++ b/docs/topics/class-based-views/generic-editing.txt @@ -90,8 +90,8 @@ class: .. code-block:: python # models.py - from django import models from django.core.urlresolvers import reverse + from django.db import models class Author(models.Model): name = models.CharField(max_length=200) @@ -102,7 +102,7 @@ class: Then we can use :class:`CreateView` and friends to do the actual work. Notice how we're just configuring the generic class-based views here; we don't have to write any logic ourselves:: - + # views.py from django.views.generic.edit import CreateView, UpdateView, DeleteView from django.core.urlresolvers import reverse_lazy @@ -134,7 +134,7 @@ Finally, we hook these new views into the URLconf:: url(r'author/(?P\d+)/$', AuthorUpdate.as_view(), name='author_update'), url(r'author/(?P\d+)/delete/$', AuthorDelete.as_view(), name='author_delete'), ) - + .. note:: These views inherit :class:`~django.views.generic.detail.SingleObjectTemplateResponseMixin` @@ -160,8 +160,8 @@ you can use a custom :class:`ModelForm` to do this. First, add the foreign key relation to the model:: # models.py - from django import models from django.contrib.auth import User + from django.db import models class Author(models.Model): name = models.CharField(max_length=200) @@ -177,7 +177,7 @@ Create a custom :class:`ModelForm` in order to exclude the # forms.py from django import forms from myapp.models import Author - + class AuthorForm(forms.ModelForm): class Meta: model = Author @@ -190,7 +190,7 @@ In the view, use the custom :attr:`form_class` and override from django.views.generic.edit import CreateView from myapp.models import Author from myapp.forms import AuthorForm - + class AuthorCreate(CreateView): form_class = AuthorForm model = Author From fa18b0ac89723f4ed6e46e744039bf375c8945a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sun, 11 Nov 2012 00:35:46 +0200 Subject: [PATCH 124/302] Some changes to SortedDict to make it faster under py2 Refs #19276 --- django/utils/datastructures.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/django/utils/datastructures.py b/django/utils/datastructures.py index d94a05dfb4..f81fb88a19 100644 --- a/django/utils/datastructures.py +++ b/django/utils/datastructures.py @@ -1,6 +1,5 @@ import copy import warnings -from types import GeneratorType from django.utils import six @@ -120,27 +119,23 @@ class SortedDict(dict): return instance def __init__(self, data=None): - if data is None: - data = {} - elif isinstance(data, GeneratorType): - # Unfortunately we need to be able to read a generator twice. Once - # to get the data into self with our super().__init__ call and a - # second time to setup keyOrder correctly - data = list(data) - super(SortedDict, self).__init__(data) - if isinstance(data, dict): - self.keyOrder = list(data) + if data is None or isinstance(data, dict): + data = data or [] + super(SortedDict, self).__init__(data) + self.keyOrder = list(data) if data else [] else: - self.keyOrder = [] - seen = set() + super(SortedDict, self).__init__() + super_set = super(SortedDict, self).__setitem__ for key, value in data: - if key not in seen: + # Take the ordering from first key + if key not in self: self.keyOrder.append(key) - seen.add(key) + # But override with last value in data (dict() does this) + super_set(key, value) def __deepcopy__(self, memo): return self.__class__([(key, copy.deepcopy(value, memo)) - for key, value in six.iteritems(self)]) + for key, value in self.items()]) def __copy__(self): # The Python's default copy implementation will alter the state @@ -199,13 +194,13 @@ class SortedDict(dict): itervalues = _itervalues def items(self): - return list(self.iteritems()) + return [(k, self[k]) for k in self.keyOrder] def keys(self): - return list(self.iterkeys()) + return self.keyOrder[:] def values(self): - return list(self.itervalues()) + return [self[k] for k in self.keyOrder] def update(self, dict_): for k, v in six.iteritems(dict_): From ce1af8d7023f02e4521cce3bcdbc9fd13d76c5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sun, 11 Nov 2012 01:59:24 +0200 Subject: [PATCH 125/302] Removed use of SortedDict for query.alias_refcount This will have a smallish impact on performance. Refs #19276. --- django/db/models/sql/query.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py index ce45ec314a..4cfb816958 100644 --- a/django/db/models/sql/query.py +++ b/django/db/models/sql/query.py @@ -103,7 +103,7 @@ class Query(object): def __init__(self, model, where=WhereNode): self.model = model - self.alias_refcount = SortedDict() + self.alias_refcount = {} # alias_map is the most important data structure regarding joins. # It's used for recording which joins exist in the query and what # type they are. The key is the alias of the joined table (possibly @@ -860,7 +860,7 @@ class Query(object): count. Note that after execution, the reference counts are zeroed, so tables added in compiler will not be seen by this method. """ - return len([1 for count in six.itervalues(self.alias_refcount) if count]) + return len([1 for count in self.alias_refcount.values() if count]) def join(self, connection, reuse=REUSE_ALL, promote=False, outer_if_first=False, nullable=False): @@ -1532,9 +1532,9 @@ class Query(object): # comparison to NULL (e.g. in # Tag.objects.exclude(parent__parent__name='t1'), a tag with no parent # would otherwise be overlooked). - active_positions = [pos for (pos, count) in - enumerate(six.itervalues(query.alias_refcount)) if count] - if active_positions[-1] > 1: + active_positions = len([count for count + in query.alias_refcount.items() if count]) + if active_positions > 1: self.add_filter(('%s__isnull' % prefix, False), negate=True, trim=True, can_reuse=can_reuse) From ebcf6b36ffa7ee20b7219d34200b093186befcb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Sun, 11 Nov 2012 14:05:29 +0200 Subject: [PATCH 126/302] Fixed select_related performance regressions The regression was caused by select_related fix for Oracle, commit c159d9cec0baab7bbd04d5d51a92a51e354a722a. --- django/db/models/options.py | 11 +++++++---- django/db/models/query.py | 27 +++++++++++---------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/django/db/models/options.py b/django/db/models/options.py index b04f3d4c2d..7ea6e4b744 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -9,11 +9,10 @@ from django.db.models.fields.related import ManyToManyRel from django.db.models.fields import AutoField, FieldDoesNotExist from django.db.models.fields.proxy import OrderWrt from django.db.models.loading import get_models, app_cache_ready -from django.utils.translation import activate, deactivate_all, get_language, string_concat -from django.utils.encoding import force_text, smart_text -from django.utils.datastructures import SortedDict from django.utils import six -from django.utils.encoding import python_2_unicode_compatible +from django.utils.datastructures import SortedDict +from django.utils.encoding import force_text, smart_text, python_2_unicode_compatible +from django.utils.translation import activate, deactivate_all, get_language, string_concat # Calculate the verbose_name by converting from InitialCaps to "lowercase with spaces". get_verbose_name = lambda class_name: re.sub('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))', ' \\1', class_name).lower().strip() @@ -175,6 +174,10 @@ class Options(object): self.pk = field field.serialize = False + def pk_index(self): + return [pos for pos, field in enumerate(self.fields) + if field == self.pk][0] + def setup_proxy(self, target): """ Does the internal setup so that the current model is a proxy for diff --git a/django/db/models/query.py b/django/db/models/query.py index a3b28e9228..67fef52f36 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -1395,8 +1395,12 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, klass_info = get_klass_info(o.model, max_depth=max_depth, cur_depth=cur_depth+1, requested=next, only_load=only_load, local_only=True) reverse_related_fields.append((o.field, klass_info)) + if field_names: + pk_idx = field_names.index(klass._meta.pk.attname) + else: + pk_idx = klass._meta.pk_index() - return klass, field_names, field_count, related_fields, reverse_related_fields + return klass, field_names, field_count, related_fields, reverse_related_fields, pk_idx def get_cached_row(row, index_start, using, klass_info, offset=0): @@ -1419,26 +1423,17 @@ def get_cached_row(row, index_start, using, klass_info, offset=0): """ if klass_info is None: return None - klass, field_names, field_count, related_fields, reverse_related_fields = klass_info + klass, field_names, field_count, related_fields, reverse_related_fields, pk_idx = klass_info fields = row[index_start : index_start + field_count] - # If all the select_related columns are None, then the related + # If the pk column is None (or the Oracle equivalent ''), then the related # object must be non-existent - set the relation to None. - # Otherwise, construct the related object. Also, some backends treat '' - # and None equivalently for char fields, so we have to be prepared for - # '' values. - if connections[using].features.interprets_empty_strings_as_nulls: - vals = tuple([None if f == '' else f for f in fields]) - else: - vals = fields - - if vals == (None,) * field_count: + if fields[pk_idx] == None or fields[pk_idx] == '': obj = None + elif field_names: + obj = klass(**dict(zip(field_names, fields))) else: - if field_names: - obj = klass(**dict(zip(field_names, fields))) - else: - obj = klass(*fields) + obj = klass(*fields) # If an object was retrieved, set the database state. if obj: From 54fbe6ce5f758d63213ef571f175978823881834 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 13 Nov 2012 13:51:50 -0800 Subject: [PATCH 127/302] Correct link to Sentry django-sentry is no longer maintained, and sentry is the replacement. --- docs/topics/logging.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/logging.txt b/docs/topics/logging.txt index 7bd56e92ec..bbcd011b59 100644 --- a/docs/topics/logging.txt +++ b/docs/topics/logging.txt @@ -470,13 +470,13 @@ Python logging module. with names and values of local variables at each level of the stack, plus the values of your Django settings. This information is potentially very sensitive, and you may not want to send it over email. Consider using - something such as `django-sentry`_ to get the best of both worlds -- the + something such as `Sentry`_ to get the best of both worlds -- the rich information of full tracebacks plus the security of *not* sending the information over email. You may also explicitly designate certain sensitive information to be filtered out of error reports -- learn more on :ref:`Filtering error reports`. -.. _django-sentry: http://pypi.python.org/pypi/django-sentry +.. _django-sentry: http://pypi.python.org/pypi/sentry Filters From 1e34fd3c03e8f9a9e2b9be35488b8209178a4df0 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Tue, 13 Nov 2012 14:48:23 -0800 Subject: [PATCH 128/302] fixed a broken link in the docs --- docs/topics/logging.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/logging.txt b/docs/topics/logging.txt index bbcd011b59..d016b59969 100644 --- a/docs/topics/logging.txt +++ b/docs/topics/logging.txt @@ -476,7 +476,7 @@ Python logging module. sensitive information to be filtered out of error reports -- learn more on :ref:`Filtering error reports`. -.. _django-sentry: http://pypi.python.org/pypi/sentry +.. _Sentry: http://pypi.python.org/pypi/sentry Filters From 1620c27936718a87bbc8acce7886ef20fee2b3fe Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 14 Nov 2012 10:39:58 +0100 Subject: [PATCH 129/302] Fixed #19186 -- Fixed sending mail with unicode content on Python 3 Thanks alex_po for the report and Luke Plant for the analysis. --- django/core/mail/backends/smtp.py | 5 ++++- tests/regressiontests/mail/tests.py | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/django/core/mail/backends/smtp.py b/django/core/mail/backends/smtp.py index 18437c6282..b6f7f560ed 100644 --- a/django/core/mail/backends/smtp.py +++ b/django/core/mail/backends/smtp.py @@ -7,6 +7,7 @@ from django.conf import settings from django.core.mail.backends.base import BaseEmailBackend from django.core.mail.utils import DNS_NAME from django.core.mail.message import sanitize_address +from django.utils.encoding import force_bytes class EmailBackend(BaseEmailBackend): @@ -102,9 +103,11 @@ class EmailBackend(BaseEmailBackend): from_email = sanitize_address(email_message.from_email, email_message.encoding) recipients = [sanitize_address(addr, email_message.encoding) for addr in email_message.recipients()] + message = email_message.message() + charset = message.get_charset().get_output_charset() if message.get_charset() else 'utf-8' try: self.connection.sendmail(from_email, recipients, - email_message.message().as_string()) + force_bytes(message.as_string(), charset)) except: if not self.fail_silently: raise diff --git a/tests/regressiontests/mail/tests.py b/tests/regressiontests/mail/tests.py index 33898cc1d5..b798cb21aa 100644 --- a/tests/regressiontests/mail/tests.py +++ b/tests/regressiontests/mail/tests.py @@ -17,6 +17,7 @@ from django.core.mail.backends import console, dummy, locmem, filebased, smtp from django.core.mail.message import BadHeaderError from django.test import TestCase from django.test.utils import override_settings +from django.utils.encoding import force_str, force_text from django.utils.six import PY3, StringIO from django.utils.translation import ugettext_lazy @@ -357,6 +358,14 @@ class BaseEmailBackendTests(object): self.assertEqual(message["from"], "from@example.com") self.assertEqual(message.get_all("to"), ["to@example.com"]) + def test_send_unicode(self): + email = EmailMessage('Chère maman', 'Je t\'aime très fort', 'from@example.com', ['to@example.com']) + num_sent = mail.get_connection().send_messages([email]) + self.assertEqual(num_sent, 1) + message = self.get_the_message() + self.assertEqual(message["subject"], '=?utf-8?q?Ch=C3=A8re_maman?=') + self.assertEqual(force_text(message.get_payload()), 'Je t\'aime très fort') + def test_send_many(self): email1 = EmailMessage('Subject', 'Content1', 'from@example.com', ['to@example.com']) email2 = EmailMessage('Subject', 'Content2', 'from@example.com', ['to@example.com']) @@ -526,8 +535,8 @@ class FileBackendTests(BaseEmailBackendTests, TestCase): messages = [] for filename in os.listdir(self.tmp_dir): with open(os.path.join(self.tmp_dir, filename), 'r') as fp: - session = fp.read().split('\n' + ('-' * 79) + '\n') - messages.extend(email.message_from_string(str(m)) for m in session if m) + session = force_text(fp.read()).split('\n' + ('-' * 79) + '\n') + messages.extend(email.message_from_string(force_str(m)) for m in session if m) return messages def test_file_sessions(self): @@ -579,8 +588,8 @@ class ConsoleBackendTests(BaseEmailBackendTests, TestCase): self.stream = sys.stdout = StringIO() def get_mailbox_content(self): - messages = self.stream.getvalue().split('\n' + ('-' * 79) + '\n') - return [email.message_from_string(str(m)) for m in messages if m] + messages = force_text(self.stream.getvalue()).split('\n' + ('-' * 79) + '\n') + return [email.message_from_string(force_str(m)) for m in messages if m] def test_console_stream_kwarg(self): """ From 550ddc66b496473c8ee282c7ab6be5885a359d75 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 14 Nov 2012 10:50:15 +0100 Subject: [PATCH 130/302] Fixed #19272 -- Fixed gettext_lazy returned type on Python 2 Thanks tyrion for the report. --- django/utils/translation/trans_real.py | 3 ++- tests/regressiontests/i18n/tests.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/django/utils/translation/trans_real.py b/django/utils/translation/trans_real.py index 9e94840ee0..1bcef2d8de 100644 --- a/django/utils/translation/trans_real.py +++ b/django/utils/translation/trans_real.py @@ -246,7 +246,8 @@ def do_translate(message, translation_function): """ global _default - eol_message = message.replace('\r\n', '\n').replace('\r', '\n') + # str() is allowing a bytestring message to remain bytestring on Python 2 + eol_message = message.replace(str('\r\n'), str('\n')).replace(str('\r'), str('\n')) t = getattr(_active, "value", None) if t is not None: result = getattr(t, translation_function)(eol_message) diff --git a/tests/regressiontests/i18n/tests.py b/tests/regressiontests/i18n/tests.py index 4054f85ef0..2e0c097a19 100644 --- a/tests/regressiontests/i18n/tests.py +++ b/tests/regressiontests/i18n/tests.py @@ -83,6 +83,10 @@ class TranslationTests(TestCase): s4 = ugettext_lazy('Some other string') self.assertEqual(False, s == s4) + if not six.PY3: + # On Python 2, gettext_lazy should not transform a bytestring to unicode + self.assertEqual(gettext_lazy(b"test").upper(), b"TEST") + def test_lazy_pickle(self): s1 = ugettext_lazy("test") self.assertEqual(six.text_type(s1), "test") From 2dbfa66f4db54c2bbac0f160de96a91fcf39997d Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Wed, 14 Nov 2012 05:46:30 -0500 Subject: [PATCH 131/302] Fixed #19289 - Removed an out of place sentence in tutorial 2. Thanks colinnkeenan for the report. --- docs/intro/tutorial02.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docs/intro/tutorial02.txt b/docs/intro/tutorial02.txt index b87b280d7c..2c8d25ae6f 100644 --- a/docs/intro/tutorial02.txt +++ b/docs/intro/tutorial02.txt @@ -445,11 +445,6 @@ live anywhere on your filesystem that Django can access. (Django runs as whatever user your server runs.) However, keeping your templates within the project is a good convention to follow. -When you’ve done that, create a directory polls in your template directory. -Within that, create a file called index.html. Note that our -``loader.get_template('polls/index.html')`` code from above maps to -[template_directory]/polls/index.html” on the filesystem. - By default, :setting:`TEMPLATE_DIRS` is empty. So, let's add a line to it, to tell Django where our templates live:: From 92d7f541da8b59520c833b19fbba52d3ecef2428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 15 Nov 2012 14:23:02 +0200 Subject: [PATCH 132/302] Fixed #19058 -- Fixed Oracle GIS crash The problem is the same as in #10888 which was reintroduced when bulk_insert was added. Thanks to Jani Tiainen for report, patch and also testing the final patch on Oracle GIS. --- .../gis/db/backends/oracle/compiler.py | 24 +------------------ .../gis/db/backends/oracle/operations.py | 9 +++++++ django/db/backends/__init__.py | 5 ++++ django/db/models/sql/compiler.py | 2 ++ 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/django/contrib/gis/db/backends/oracle/compiler.py b/django/contrib/gis/db/backends/oracle/compiler.py index f0eb5cad00..98da0163ba 100644 --- a/django/contrib/gis/db/backends/oracle/compiler.py +++ b/django/contrib/gis/db/backends/oracle/compiler.py @@ -7,29 +7,7 @@ class GeoSQLCompiler(BaseGeoSQLCompiler, SQLCompiler): pass class SQLInsertCompiler(compiler.SQLInsertCompiler, GeoSQLCompiler): - def placeholder(self, field, val): - if field is None: - # A field value of None means the value is raw. - return val - elif hasattr(field, 'get_placeholder'): - # Some fields (e.g. geo fields) need special munging before - # they can be inserted. - ph = field.get_placeholder(val, self.connection) - if ph == 'NULL': - # If the placeholder returned is 'NULL', then we need to - # to remove None from the Query parameters. Specifically, - # cx_Oracle will assume a CHAR type when a placeholder ('%s') - # is used for columns of MDSYS.SDO_GEOMETRY. Thus, we use - # 'NULL' for the value, and remove None from the query params. - # See also #10888. - param_idx = self.query.columns.index(field.column) - params = list(self.query.params) - params.pop(param_idx) - self.query.params = tuple(params) - return ph - else: - # Return the common case for the placeholder - return '%s' + pass class SQLDeleteCompiler(compiler.SQLDeleteCompiler, GeoSQLCompiler): pass diff --git a/django/contrib/gis/db/backends/oracle/operations.py b/django/contrib/gis/db/backends/oracle/operations.py index 35a4d9491d..4e42b4cf00 100644 --- a/django/contrib/gis/db/backends/oracle/operations.py +++ b/django/contrib/gis/db/backends/oracle/operations.py @@ -288,3 +288,12 @@ class OracleOperations(DatabaseOperations, BaseSpatialOperations): def spatial_ref_sys(self): from django.contrib.gis.db.backends.oracle.models import SpatialRefSys return SpatialRefSys + + def modify_insert_params(self, placeholders, params): + """Drop out insert parameters for NULL placeholder. Needed for Oracle Spatial + backend due to #10888 + """ + # This code doesn't work for bulk insert cases. + assert len(placeholders) == 1 + return [[param for pholder,param + in six.moves.zip(placeholders[0], params[0]) if pholder != 'NULL'], ] diff --git a/django/db/backends/__init__.py b/django/db/backends/__init__.py index 02d2a16a46..8dd0212dbb 100644 --- a/django/db/backends/__init__.py +++ b/django/db/backends/__init__.py @@ -916,6 +916,11 @@ class BaseDatabaseOperations(object): conn = ' %s ' % connector return conn.join(sub_expressions) + def modify_insert_params(self, placeholders, params): + """Allow modification of insert parameters. Needed for Oracle Spatial + backend due to #10888. + """ + return params class BaseDatabaseIntrospection(object): """ diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index f0a1611e9c..2ad4542341 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -856,6 +856,8 @@ class SQLInsertCompiler(SQLCompiler): [self.placeholder(field, v) for field, v in zip(fields, val)] for val in values ] + # Oracle Spatial needs to remove some values due to #10888 + params = self.connection.ops.modify_insert_params(placeholders, params) if self.return_id and self.connection.features.can_return_id_from_insert: params = params[0] col = "%s.%s" % (qn(opts.db_table), qn(opts.pk.column)) From f51e409a5fb34020e170494320a421503689aea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Fri, 9 Nov 2012 20:21:46 +0200 Subject: [PATCH 133/302] Fixed #13781 -- Improved select_related in inheritance situations The select_related code got confused when it needed to travel a reverse relation to a model which had different parent than the originally travelled relation. Thanks to Trac aliases shauncutts for report and ungenio for original patch (committed patch is somewhat modified version of that). --- django/db/models/options.py | 6 +- django/db/models/query.py | 83 ++++++++------ django/db/models/sql/compiler.py | 13 ++- .../select_related_onetoone/models.py | 42 ++++++++ .../select_related_onetoone/tests.py | 102 +++++++++++++++++- 5 files changed, 203 insertions(+), 43 deletions(-) diff --git a/django/db/models/options.py b/django/db/models/options.py index 7ea6e4b744..ab2f44e2f7 100644 --- a/django/db/models/options.py +++ b/django/db/models/options.py @@ -75,6 +75,7 @@ class Options(object): from django.db.backends.util import truncate_name cls._meta = self + self.model = cls self.installed = re.sub('\.models$', '', cls.__module__) in settings.INSTALLED_APPS # First, construct the default values for these options. self.object_name = cls.__name__ @@ -464,7 +465,7 @@ class Options(object): a granparent or even more distant relation. """ if not self.parents: - return + return None if model in self.parents: return [model] for parent in self.parents: @@ -472,8 +473,7 @@ class Options(object): if res: res.insert(0, parent) return res - raise TypeError('%r is not an ancestor of this model' - % model._meta.module_name) + return None def get_parent_list(self): """ diff --git a/django/db/models/query.py b/django/db/models/query.py index 67fef52f36..d5379a5f6a 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -1300,7 +1300,7 @@ class EmptyQuerySet(QuerySet): value_annotation = False def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, - only_load=None, local_only=False): + only_load=None, from_parent=None): """ Helper function that recursively returns an information for a klass, to be used in get_cached_row. It exists just to compute this information only @@ -1320,8 +1320,10 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, * only_load - if the query has had only() or defer() applied, this is the list of field names that will be returned. If None, the full field list for `klass` can be assumed. - * local_only - Only populate local fields. This is used when - following reverse select-related relations + * from_parent - the parent model used to get to this model + + Note that when travelling from parent to child, we will only load child + fields which aren't in the parent. """ if max_depth and requested is None and cur_depth > max_depth: # We've recursed deeply enough; stop now. @@ -1347,7 +1349,9 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, for field, model in klass._meta.get_fields_with_model(): if field.name not in load_fields: skip.add(field.attname) - elif local_only and model is not None: + elif from_parent and issubclass(from_parent, model.__class__): + # Avoid loading fields already loaded for parent model for + # child models. continue else: init_list.append(field.attname) @@ -1361,16 +1365,22 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, else: # Load all fields on klass - # We trying to not populate field_names variable for perfomance reason. - # If field_names variable is set, it is used to instantiate desired fields, - # by passing **dict(zip(field_names, fields)) as kwargs to Model.__init__ method. - # But kwargs version of Model.__init__ is slower, so we should avoid using - # it when it is not really neccesary. - if local_only and len(klass._meta.local_fields) != len(klass._meta.fields): - field_count = len(klass._meta.local_fields) - field_names = [f.attname for f in klass._meta.local_fields] - else: - field_count = len(klass._meta.fields) + field_count = len(klass._meta.fields) + # Check if we need to skip some parent fields. + if from_parent and len(klass._meta.local_fields) != len(klass._meta.fields): + # Only load those fields which haven't been already loaded into + # 'from_parent'. + non_seen_models = [p for p in klass._meta.get_parent_list() + if not issubclass(from_parent, p)] + # Load local fields, too... + non_seen_models.append(klass) + field_names = [f.attname for f in klass._meta.fields + if f.model in non_seen_models] + field_count = len(field_names) + # Try to avoid populating field_names variable for perfomance reasons. + # If field_names variable is set, we use **kwargs based model init + # which is slower than normal init. + if field_count == len(klass._meta.fields): field_names = () restricted = requested is not None @@ -1392,8 +1402,9 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, if o.field.unique and select_related_descend(o.field, restricted, requested, only_load.get(o.model), reverse=True): next = requested[o.field.related_query_name()] + parent = klass if issubclass(o.model, klass) else None klass_info = get_klass_info(o.model, max_depth=max_depth, cur_depth=cur_depth+1, - requested=next, only_load=only_load, local_only=True) + requested=next, only_load=only_load, from_parent=parent) reverse_related_fields.append((o.field, klass_info)) if field_names: pk_idx = field_names.index(klass._meta.pk.attname) @@ -1403,7 +1414,8 @@ def get_klass_info(klass, max_depth=0, cur_depth=0, requested=None, return klass, field_names, field_count, related_fields, reverse_related_fields, pk_idx -def get_cached_row(row, index_start, using, klass_info, offset=0): +def get_cached_row(row, index_start, using, klass_info, offset=0, + parent_data=()): """ Helper function that recursively returns an object with the specified related attributes already populated. @@ -1418,13 +1430,16 @@ def get_cached_row(row, index_start, using, klass_info, offset=0): * offset - the number of additional fields that are known to exist in row for `klass`. This usually means the number of annotated results on `klass`. - * using - the database alias on which the query is being executed. + * using - the database alias on which the query is being executed. * klass_info - result of the get_klass_info function + * parent_data - parent model data in format (field, value). Used + to populate the non-local fields of child models. """ if klass_info is None: return None klass, field_names, field_count, related_fields, reverse_related_fields, pk_idx = klass_info + fields = row[index_start : index_start + field_count] # If the pk column is None (or the Oracle equivalent ''), then the related # object must be non-existent - set the relation to None. @@ -1434,7 +1449,6 @@ def get_cached_row(row, index_start, using, klass_info, offset=0): obj = klass(**dict(zip(field_names, fields))) else: obj = klass(*fields) - # If an object was retrieved, set the database state. if obj: obj._state.db = using @@ -1464,34 +1478,35 @@ def get_cached_row(row, index_start, using, klass_info, offset=0): # Only handle the restricted case - i.e., don't do a depth # descent into reverse relations unless explicitly requested for f, klass_info in reverse_related_fields: + # Transfer data from this object to childs. + parent_data = [] + for rel_field, rel_model in klass_info[0]._meta.get_fields_with_model(): + if rel_model is not None and isinstance(obj, rel_model): + parent_data.append((rel_field, getattr(obj, rel_field.attname))) # Recursively retrieve the data for the related object - cached_row = get_cached_row(row, index_end, using, klass_info) + cached_row = get_cached_row(row, index_end, using, klass_info, + parent_data=parent_data) # If the recursive descent found an object, populate the # descriptor caches relevant to the object if cached_row: rel_obj, index_end = cached_row if obj is not None: - # If the field is unique, populate the - # reverse descriptor cache + # populate the reverse descriptor cache setattr(obj, f.related.get_cache_name(), rel_obj) if rel_obj is not None: # If the related object exists, populate # the descriptor cache. setattr(rel_obj, f.get_cache_name(), obj) - # Now populate all the non-local field values - # on the related object - for rel_field, rel_model in rel_obj._meta.get_fields_with_model(): - if rel_model is not None: + # Populate related object caches using parent data. + for rel_field, _ in parent_data: + if rel_field.rel: setattr(rel_obj, rel_field.attname, getattr(obj, rel_field.attname)) - # populate the field cache for any related object - # that has already been retrieved - if rel_field.rel: - try: - cached_obj = getattr(obj, rel_field.get_cache_name()) - setattr(rel_obj, rel_field.get_cache_name(), cached_obj) - except AttributeError: - # Related object hasn't been cached yet - pass + try: + cached_obj = getattr(obj, rel_field.get_cache_name()) + setattr(rel_obj, rel_field.get_cache_name(), cached_obj) + except AttributeError: + # Related object hasn't been cached yet + pass return obj, index_end diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py index 2ad4542341..8cfb12a8e3 100644 --- a/django/db/models/sql/compiler.py +++ b/django/db/models/sql/compiler.py @@ -240,7 +240,7 @@ class SQLCompiler(object): return result def get_default_columns(self, with_aliases=False, col_aliases=None, - start_alias=None, opts=None, as_pairs=False, local_only=False): + start_alias=None, opts=None, as_pairs=False, from_parent=None): """ Computes the default columns for selecting every field in the base model. Will sometimes be called to pull in related models (e.g. via @@ -265,7 +265,8 @@ class SQLCompiler(object): if start_alias: seen = {None: start_alias} for field, model in opts.get_fields_with_model(): - if local_only and model is not None: + if from_parent and model is not None and issubclass(from_parent, model): + # Avoid loading data for already loaded parents. continue if start_alias: try: @@ -686,11 +687,13 @@ class SQLCompiler(object): (alias, table, f.rel.get_related_field().column, f.column), promote=True ) + from_parent = (opts.model if issubclass(model, opts.model) + else None) columns, aliases = self.get_default_columns(start_alias=alias, - opts=model._meta, as_pairs=True, local_only=True) + opts=model._meta, as_pairs=True, from_parent=from_parent) self.query.related_select_cols.extend( - SelectInfo(col, field) for col, field in zip(columns, model._meta.fields)) - + SelectInfo(col, field) for col, field + in zip(columns, model._meta.fields)) next = requested.get(f.related_query_name(), {}) # Use True here because we are looking at the _reverse_ side of # the relation, which is always nullable. diff --git a/tests/regressiontests/select_related_onetoone/models.py b/tests/regressiontests/select_related_onetoone/models.py index 3284defb11..d32faafbb9 100644 --- a/tests/regressiontests/select_related_onetoone/models.py +++ b/tests/regressiontests/select_related_onetoone/models.py @@ -51,6 +51,7 @@ class StatDetails(models.Model): class AdvancedUserStat(UserStat): karma = models.IntegerField() + class Image(models.Model): name = models.CharField(max_length=100) @@ -58,3 +59,44 @@ class Image(models.Model): class Product(models.Model): name = models.CharField(max_length=100) image = models.OneToOneField(Image, null=True) + + +@python_2_unicode_compatible +class Parent1(models.Model): + name1 = models.CharField(max_length=50) + + def __str__(self): + return self.name1 + + +@python_2_unicode_compatible +class Parent2(models.Model): + # Avoid having two "id" fields in the Child1 subclass + id2 = models.AutoField(primary_key=True) + name2 = models.CharField(max_length=50) + + def __str__(self): + return self.name2 + + +@python_2_unicode_compatible +class Child1(Parent1, Parent2): + value = models.IntegerField() + + def __str__(self): + return self.name1 + + +@python_2_unicode_compatible +class Child2(Parent1): + parent2 = models.OneToOneField(Parent2) + value = models.IntegerField() + + def __str__(self): + return self.name1 + +class Child3(Child2): + value3 = models.IntegerField() + +class Child4(Child1): + value4 = models.IntegerField() diff --git a/tests/regressiontests/select_related_onetoone/tests.py b/tests/regressiontests/select_related_onetoone/tests.py index 1373f04717..d4a1275e49 100644 --- a/tests/regressiontests/select_related_onetoone/tests.py +++ b/tests/regressiontests/select_related_onetoone/tests.py @@ -1,9 +1,11 @@ from __future__ import absolute_import from django.test import TestCase +from django.utils import unittest from .models import (User, UserProfile, UserStat, UserStatResult, StatDetails, - AdvancedUserStat, Image, Product) + AdvancedUserStat, Image, Product, Parent1, Parent2, Child1, Child2, Child3, + Child4) class ReverseSelectRelatedTestCase(TestCase): @@ -21,6 +23,14 @@ class ReverseSelectRelatedTestCase(TestCase): advstat = AdvancedUserStat.objects.create(user=user2, posts=200, karma=5, results=results2) StatDetails.objects.create(base_stats=advstat, comments=250) + p1 = Parent1(name1="Only Parent1") + p1.save() + c1 = Child1(name1="Child1 Parent1", name2="Child1 Parent2", value=1) + c1.save() + p2 = Parent2(name2="Child2 Parent2") + p2.save() + c2 = Child2(name1="Child2 Parent1", parent2=p2, value=2) + c2.save() def test_basic(self): with self.assertNumQueries(1): @@ -108,3 +118,93 @@ class ReverseSelectRelatedTestCase(TestCase): image = Image.objects.select_related('product').get() with self.assertRaises(Product.DoesNotExist): image.product + + def test_parent_only(self): + with self.assertNumQueries(1): + p = Parent1.objects.select_related('child1').get(name1="Only Parent1") + with self.assertNumQueries(0): + with self.assertRaises(Child1.DoesNotExist): + p.child1 + + def test_multiple_subclass(self): + with self.assertNumQueries(1): + p = Parent1.objects.select_related('child1').get(name1="Child1 Parent1") + self.assertEqual(p.child1.name2, 'Child1 Parent2') + + def test_onetoone_with_subclass(self): + with self.assertNumQueries(1): + p = Parent2.objects.select_related('child2').get(name2="Child2 Parent2") + self.assertEqual(p.child2.name1, 'Child2 Parent1') + + def test_onetoone_with_two_subclasses(self): + with self.assertNumQueries(1): + p = Parent2.objects.select_related('child2', "child2__child3").get(name2="Child2 Parent2") + self.assertEqual(p.child2.name1, 'Child2 Parent1') + with self.assertRaises(Child3.DoesNotExist): + p.child2.child3 + p3 = Parent2(name2="Child3 Parent2") + p3.save() + c2 = Child3(name1="Child3 Parent1", parent2=p3, value=2, value3=3) + c2.save() + with self.assertNumQueries(1): + p = Parent2.objects.select_related('child2', "child2__child3").get(name2="Child3 Parent2") + self.assertEqual(p.child2.name1, 'Child3 Parent1') + self.assertEqual(p.child2.child3.value3, 3) + self.assertEqual(p.child2.child3.value, p.child2.value) + self.assertEqual(p.child2.name1, p.child2.child3.name1) + + def test_multiinheritance_two_subclasses(self): + with self.assertNumQueries(1): + p = Parent1.objects.select_related('child1', 'child1__child4').get(name1="Child1 Parent1") + self.assertEqual(p.child1.name2, 'Child1 Parent2') + self.assertEqual(p.child1.name1, p.name1) + with self.assertRaises(Child4.DoesNotExist): + p.child1.child4 + Child4(name1='n1', name2='n2', value=1, value4=4).save() + with self.assertNumQueries(1): + p = Parent2.objects.select_related('child1', 'child1__child4').get(name2="n2") + self.assertEqual(p.name2, 'n2') + self.assertEqual(p.child1.name1, 'n1') + self.assertEqual(p.child1.name2, p.name2) + self.assertEqual(p.child1.value, 1) + self.assertEqual(p.child1.child4.name1, p.child1.name1) + self.assertEqual(p.child1.child4.name2, p.child1.name2) + self.assertEqual(p.child1.child4.value, p.child1.value) + self.assertEqual(p.child1.child4.value4, 4) + + @unittest.expectedFailure + def test_inheritance_deferred(self): + c = Child4.objects.create(name1='n1', name2='n2', value=1, value4=4) + with self.assertNumQueries(1): + p = Parent2.objects.select_related('child1').only( + 'id2', 'child1__value').get(name2="n2") + self.assertEqual(p.id2, c.id2) + self.assertEqual(p.child1.value, 1) + p = Parent2.objects.select_related('child1').only( + 'id2', 'child1__value').get(name2="n2") + with self.assertNumQueries(1): + self.assertEquals(p.name2, 'n2') + p = Parent2.objects.select_related('child1').only( + 'id2', 'child1__value').get(name2="n2") + with self.assertNumQueries(1): + self.assertEquals(p.child1.name2, 'n2') + + @unittest.expectedFailure + def test_inheritance_deferred2(self): + c = Child4.objects.create(name1='n1', name2='n2', value=1, value4=4) + qs = Parent2.objects.select_related('child1', 'child4').only( + 'id2', 'child1__value', 'child1__child4__value4') + with self.assertNumQueries(1): + p = qs.get(name2="n2") + self.assertEqual(p.id2, c.id2) + self.assertEqual(p.child1.value, 1) + self.assertEqual(p.child1.child4.value4, 4) + self.assertEqual(p.child1.child4.id2, c.id2) + p = qs.get(name2="n2") + with self.assertNumQueries(1): + self.assertEquals(p.child1.name2, 'n2') + p = qs.get(name2="n2") + with self.assertNumQueries(1): + self.assertEquals(p.child1.name1, 'n1') + with self.assertNumQueries(1): + self.assertEquals(p.child1.child4.name1, 'n1') From 71e14cf3aa024496adcb23e83ddf13a7c5ddeb32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Fri, 8 Jun 2012 15:57:42 +0300 Subject: [PATCH 134/302] Fixed #18347 -- Removed autofield raw SQL inserts from tests --- .../transactions_regress/tests.py | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/regressiontests/transactions_regress/tests.py b/tests/regressiontests/transactions_regress/tests.py index 472e2aafd9..66e047799e 100644 --- a/tests/regressiontests/transactions_regress/tests.py +++ b/tests/regressiontests/transactions_regress/tests.py @@ -24,17 +24,15 @@ class TestTransactionClosing(TransactionTestCase): def raw_sql(): "Write a record using raw sql under a commit_on_success decorator" cursor = connection.cursor() - cursor.execute("INSERT into transactions_regress_mod (id,fld) values (17,18)") + cursor.execute("INSERT into transactions_regress_mod (fld) values (18)") raw_sql() # Rollback so that if the decorator didn't commit, the record is unwritten transaction.rollback() - try: - # Check that the record is in the DB - obj = Mod.objects.get(pk=17) - self.assertEqual(obj.fld, 18) - except Mod.DoesNotExist: - self.fail("transaction with raw sql not committed") + self.assertEqual(Mod.objects.count(), 1) + # Check that the record is in the DB + obj = Mod.objects.all()[0] + self.assertEqual(obj.fld, 18) def test_commit_manually_enforced(self): """ @@ -115,19 +113,16 @@ class TestTransactionClosing(TransactionTestCase): be committed. """ cursor = connection.cursor() - cursor.execute("INSERT into transactions_regress_mod (id,fld) values (1,2)") + cursor.execute("INSERT into transactions_regress_mod (fld) values (2)") transaction.rollback() - cursor.execute("INSERT into transactions_regress_mod (id,fld) values (1,2)") + cursor.execute("INSERT into transactions_regress_mod (fld) values (2)") reuse_cursor_ref() # Rollback so that if the decorator didn't commit, the record is unwritten transaction.rollback() - try: - # Check that the record is in the DB - obj = Mod.objects.get(pk=1) - self.assertEqual(obj.fld, 2) - except Mod.DoesNotExist: - self.fail("After ending a transaction, cursor use no longer sets dirty") + self.assertEqual(Mod.objects.count(), 1) + obj = Mod.objects.all()[0] + self.assertEqual(obj.fld, 2) def test_failing_query_transaction_closed(self): """ From 1194a9699932088385f9f88869be28a251597f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 15 Nov 2012 20:17:57 +0200 Subject: [PATCH 135/302] Fixed a regression in select_related The regression was caused by the fix for #13781 (commit f51e409a5fb34020e170494320a421503689aea0). Reason was leaving off some crucial lines when resolving a merge conflict. --- django/db/models/query.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django/db/models/query.py b/django/db/models/query.py index d5379a5f6a..130be9ad96 100644 --- a/django/db/models/query.py +++ b/django/db/models/query.py @@ -1446,6 +1446,10 @@ def get_cached_row(row, index_start, using, klass_info, offset=0, if fields[pk_idx] == None or fields[pk_idx] == '': obj = None elif field_names: + fields = list(fields) + for rel_field, value in parent_data: + field_names.append(rel_field.attname) + fields.append(value) obj = klass(**dict(zip(field_names, fields))) else: obj = klass(*fields) From 7cfb567e457379f52a80fe0e8d98dd8191391c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anssi=20K=C3=A4=C3=A4ri=C3=A4inen?= Date: Thu, 15 Nov 2012 21:33:56 +0200 Subject: [PATCH 136/302] Another regression fix for select_related handling This time gis compiler.get_default_columns() wasn't up to date. Thanks to CI another regression fixed. Refs #13781 --- django/contrib/gis/db/models/sql/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/contrib/gis/db/models/sql/compiler.py b/django/contrib/gis/db/models/sql/compiler.py index 0dcf50d32a..81a9941c9e 100644 --- a/django/contrib/gis/db/models/sql/compiler.py +++ b/django/contrib/gis/db/models/sql/compiler.py @@ -101,7 +101,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): return result def get_default_columns(self, with_aliases=False, col_aliases=None, - start_alias=None, opts=None, as_pairs=False, local_only=False): + start_alias=None, opts=None, as_pairs=False, from_parent=None): """ Computes the default columns for selecting every field in the base model. Will sometimes be called to pull in related models (e.g. via @@ -127,7 +127,7 @@ class GeoSQLCompiler(compiler.SQLCompiler): if start_alias: seen = {None: start_alias} for field, model in opts.get_fields_with_model(): - if local_only and model is not None: + if from_parent and model is not None and issubclass(from_parent, model): continue if start_alias: try: From 5e3b99d4cfc50857566688c543866c82ff51dc4d Mon Sep 17 00:00:00 2001 From: vanschelven Date: Thu, 15 Nov 2012 22:53:08 +0100 Subject: [PATCH 137/302] Update django/contrib/flatpages/forms.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Translate before passing in the variables (i.e. close the bracket earlier). If you translate after passing in the variables the translated value will contain the actual values for 'url' and 'site' and will therefore not be translated. (I'm not actually using django's flatpages, but I ran into this apparent error when doing a global grep on my own project) --- django/contrib/flatpages/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/contrib/flatpages/forms.py b/django/contrib/flatpages/forms.py index e0a63e4323..a848875a9f 100644 --- a/django/contrib/flatpages/forms.py +++ b/django/contrib/flatpages/forms.py @@ -35,7 +35,7 @@ class FlatpageForm(forms.ModelForm): for site in sites: if same_url.filter(sites=site).exists(): raise forms.ValidationError( - _('Flatpage with url %(url)s already exists for site %(site)s' % - {'url': url, 'site': site})) + _('Flatpage with url %(url)s already exists for site %(site)s') % + {'url': url, 'site': site}) return super(FlatpageForm, self).clean() From ff0d3126afbc30ae1aab3a9d352300e59937fe5e Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Fri, 16 Nov 2012 14:25:45 +0100 Subject: [PATCH 138/302] Fixed #19296 -- Applied test connection sharing for spatialite Thanks pegler at gmail.com for the report and the initial patch. --- django/test/testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django/test/testcases.py b/django/test/testcases.py index 1239275264..96178b4c86 100644 --- a/django/test/testcases.py +++ b/django/test/testcases.py @@ -1077,7 +1077,7 @@ class LiveServerTestCase(TransactionTestCase): for conn in connections.all(): # If using in-memory sqlite databases, pass the connections to # the server thread. - if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' + if (conn.settings_dict['ENGINE'].rsplit('.', 1)[-1] in ('sqlite3', 'spatialite') and conn.settings_dict['NAME'] == ':memory:'): # Explicitly enable thread-shareability for this connection conn.allow_thread_sharing = True @@ -1129,7 +1129,7 @@ class LiveServerTestCase(TransactionTestCase): # Restore sqlite connections' non-sharability for conn in connections.all(): - if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' + if (conn.settings_dict['ENGINE'].rsplit('.', 1)[-1] in ('sqlite3', 'spatialite') and conn.settings_dict['NAME'] == ':memory:'): conn.allow_thread_sharing = False From d8ee46afff913975404887cdd2eec635a03013f8 Mon Sep 17 00:00:00 2001 From: Brandon Adams Date: Mon, 12 Nov 2012 17:17:05 -0500 Subject: [PATCH 139/302] comment_will_be_sent can cause a 400, not a 403 Doc cleanup for django.contrib.comments.signals.comment_will_be_sent If a receiver returns False, an HttpResponse with status code 400 is returned. A test case already exists confirming this behavior. Updated docs to reflect reality. --- django/contrib/comments/signals.py | 2 +- docs/ref/contrib/comments/signals.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django/contrib/comments/signals.py b/django/contrib/comments/signals.py index fe1083bd14..079afaf03a 100644 --- a/django/contrib/comments/signals.py +++ b/django/contrib/comments/signals.py @@ -6,7 +6,7 @@ from django.dispatch import Signal # Sent just before a comment will be posted (after it's been approved and # moderated; this can be used to modify the comment (in place) with posting # details or other such actions. If any receiver returns False the comment will be -# discarded and a 403 (not allowed) response. This signal is sent at more or less +# discarded and a 400 response. This signal is sent at more or less # the same time (just before, actually) as the Comment object's pre-save signal, # except that the HTTP request is sent along with this signal. comment_will_be_posted = Signal(providing_args=["comment", "request"]) diff --git a/docs/ref/contrib/comments/signals.txt b/docs/ref/contrib/comments/signals.txt index 9d7c435927..8274539ed7 100644 --- a/docs/ref/contrib/comments/signals.txt +++ b/docs/ref/contrib/comments/signals.txt @@ -20,8 +20,8 @@ Sent just before a comment will be saved, after it's been sanity checked and submitted. This can be used to modify the comment (in place) with posting details or other such actions. -If any receiver returns ``False`` the comment will be discarded and a 403 (not -allowed) response will be returned. +If any receiver returns ``False`` the comment will be discarded and a 400 +response will be returned. This signal is sent at more or less the same time (just before, actually) as the ``Comment`` object's :data:`~django.db.models.signals.pre_save` signal. From 44046e8a38225067d4d0feac35367eeae133446a Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Fri, 16 Nov 2012 16:50:50 -0800 Subject: [PATCH 140/302] Fixed #18985 -- made DeprecationWarnings loud Capture warnings in Python >= 2.7 and route through console handler, which is subject to DEBUG==True Thanks to dstufft for the idea, and claudep for initial patch --- django/conf/__init__.py | 10 +++++++ django/utils/log.py | 3 ++ docs/releases/1.5.txt | 7 +++++ tests/regressiontests/logging_tests/tests.py | 29 ++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/django/conf/__init__.py b/django/conf/__init__.py index 4e6f0f9211..dec4cf9418 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -6,6 +6,7 @@ variable, and then from django.conf.global_settings; see the global settings fil a list of all possible variables. """ +import logging import os import time # Needed for Windows import warnings @@ -55,6 +56,15 @@ class LazySettings(LazyObject): """ Setup logging from LOGGING_CONFIG and LOGGING settings. """ + try: + # Route warnings through python logging + logging.captureWarnings(True) + # Allow DeprecationWarnings through the warnings filters + warnings.simplefilter("default", DeprecationWarning) + except AttributeError: + # No captureWarnings on Python 2.6, DeprecationWarnings are on anyway + pass + if self.LOGGING_CONFIG: from django.utils.log import DEFAULT_LOGGING # First find the logging configuration function ... diff --git a/django/utils/log.py b/django/utils/log.py index 342ca1fc10..4806527e84 100644 --- a/django/utils/log.py +++ b/django/utils/log.py @@ -62,6 +62,9 @@ DEFAULT_LOGGING = { 'level': 'ERROR', 'propagate': False, }, + 'py.warnings': { + 'handlers': ['console'], + }, } } diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index 8b7af2cf36..c53518feaa 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -306,6 +306,13 @@ Django 1.5 also includes several smaller improvements worth noting: :attr:`~django.db.models.Options.index_together` documentation for more information. +* During Django's logging configuration verbose Deprecation warnings are + enabled and warnings are captured into the logging system. Logged warnings + are routed through the ``console`` logging handler, which by default requires + :setting:`DEBUG` to be True for output to be generated. The result is that + DeprecationWarnings should be printed to the console in development + environments the way they have been in Python versions < 2.7. + Backwards incompatible changes in 1.5 ===================================== diff --git a/tests/regressiontests/logging_tests/tests.py b/tests/regressiontests/logging_tests/tests.py index 96f81981c6..0e56195c41 100644 --- a/tests/regressiontests/logging_tests/tests.py +++ b/tests/regressiontests/logging_tests/tests.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import copy import logging +import sys import warnings from django.conf import compat_patch_logging_config, LazySettings @@ -10,9 +11,11 @@ from django.test import TestCase, RequestFactory from django.test.utils import override_settings from django.utils.log import CallbackFilter, RequireDebugFalse from django.utils.six import StringIO +from django.utils.unittest import skipUnless from ..admin_scripts.tests import AdminScriptTestCase +PYVERS = sys.version_info[:2] # logging config prior to using filter with mail_admins OLD_LOGGING = { @@ -131,6 +134,32 @@ class DefaultLoggingTest(TestCase): self.logger.error("Hey, this is an error.") self.assertEqual(output.getvalue(), 'Hey, this is an error.\n') +@skipUnless(PYVERS > (2,6), "warnings captured only in Python >= 2.7") +class WarningLoggerTests(TestCase): + """ + Tests that warnings output for DeprecationWarnings is enabled + and captured to the logging system + """ + def setUp(self): + self.logger = logging.getLogger('py.warnings') + self.old_stream = self.logger.handlers[0].stream + + def tearDown(self): + self.logger.handlers[0].stream = self.old_stream + + @override_settings(DEBUG=True) + def test_warnings_capture(self): + output = StringIO() + self.logger.handlers[0].stream = output + warnings.warn('Foo Deprecated', DeprecationWarning) + self.assertTrue('Foo Deprecated' in output.getvalue()) + + def test_warnings_capture_debug_false(self): + output = StringIO() + self.logger.handlers[0].stream = output + warnings.warn('Foo Deprecated', DeprecationWarning) + self.assertFalse('Foo Deprecated' in output.getvalue()) + class CallbackFilterTest(TestCase): def test_sense(self): From ac4aa8a76c5fe49d3a028d74c86b6589ead8d608 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Sat, 17 Nov 2012 06:49:28 -0500 Subject: [PATCH 141/302] Documented that contrib.sites creates a default site. Thanks Lorin Hochstein for the patch. --- docs/ref/contrib/sites.txt | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/ref/contrib/sites.txt b/docs/ref/contrib/sites.txt index 790e003453..7e5448b3d3 100644 --- a/docs/ref/contrib/sites.txt +++ b/docs/ref/contrib/sites.txt @@ -127,8 +127,10 @@ For example:: def my_view(request): if settings.SITE_ID == 3: # Do something. + pass else: # Do something else. + pass Of course, it's ugly to hard-code the site IDs like that. This sort of hard-coding is best for hackish fixes that you need done quickly. The @@ -141,11 +143,13 @@ domain:: current_site = get_current_site(request) if current_site.domain == 'foo.com': # Do something + pass else: # Do something else. + pass -This has also the advantage of checking if the sites framework is installed, and -return a :class:`RequestSite` instance if it is not. +This has also the advantage of checking if the sites framework is installed, +and return a :class:`RequestSite` instance if it is not. If you don't have access to the request object, you can use the ``get_current()`` method of the :class:`~django.contrib.sites.models.Site` @@ -158,8 +162,10 @@ the :setting:`SITE_ID` setting. This example is equivalent to the previous one:: current_site = Site.objects.get_current() if current_site.domain == 'foo.com': # Do something + pass else: # Do something else. + pass Getting the current domain for display -------------------------------------- @@ -200,8 +206,8 @@ subscribing to LJWorld.com alerts." Same goes for the email's message body. Note that an even more flexible (but more heavyweight) way of doing this would be to use Django's template system. Assuming Lawrence.com and LJWorld.com have -different template directories (:setting:`TEMPLATE_DIRS`), you could simply farm out -to the template system like so:: +different template directories (:setting:`TEMPLATE_DIRS`), you could simply +farm out to the template system like so:: from django.core.mail import send_mail from django.template import loader, Context @@ -216,9 +222,9 @@ to the template system like so:: # ... -In this case, you'd have to create :file:`subject.txt` and :file:`message.txt` template -files for both the LJWorld.com and Lawrence.com template directories. That -gives you more flexibility, but it's also more complex. +In this case, you'd have to create :file:`subject.txt` and :file:`message.txt` +template files for both the LJWorld.com and Lawrence.com template directories. +That gives you more flexibility, but it's also more complex. It's a good idea to exploit the :class:`~django.contrib.sites.models.Site` objects as much as possible, to remove unneeded complexity and redundancy. @@ -240,6 +246,15 @@ To do this, you can use the sites framework. A simple example:: >>> 'http://%s%s' % (Site.objects.get_current().domain, obj.get_absolute_url()) 'http://example.com/mymodel/objects/3/' + +Default site and ``syncdb`` +=========================== + +``django.contrib.sites`` registers a +:data:`~django.db.models.signals.post_syncdb` signal handler which creates a +default site named ``example.com`` with the domain ``example.com``. For +example, this site will be created after Django creates the test database. + Caching the current ``Site`` object =================================== From 4a5e8087ac7676ef08e76275c1f756778b39c13e Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Sat, 17 Nov 2012 15:37:30 +0100 Subject: [PATCH 142/302] Fixed #19136 -- Properly escape gettext context prefixes in the i18n JavaScript view template. --- django/views/i18n.py | 8 +-- .../views/locale/de/LC_MESSAGES/djangojs.mo | Bin 0 -> 615 bytes .../views/locale/de/LC_MESSAGES/djangojs.po | 40 ++++++++++++++ .../views/templates/jsi18n.html | 44 +++++++++++++++ tests/regressiontests/views/tests/__init__.py | 2 +- tests/regressiontests/views/tests/i18n.py | 52 +++++++++++++++++- tests/regressiontests/views/urls.py | 7 +++ tests/regressiontests/views/views.py | 3 + 8 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.mo create mode 100644 tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.po create mode 100644 tests/regressiontests/views/templates/jsi18n.html diff --git a/django/views/i18n.py b/django/views/i18n.py index 00ef224254..96643f0d7d 100644 --- a/django/views/i18n.py +++ b/django/views/i18n.py @@ -99,16 +99,16 @@ function ngettext(singular, plural, count) { function gettext_noop(msgid) { return msgid; } function pgettext(context, msgid) { - var value = gettext(context + '\x04' + msgid); - if (value.indexOf('\x04') != -1) { + var value = gettext(context + '\\x04' + msgid); + if (value.indexOf('\\x04') != -1) { value = msgid; } return value; } function npgettext(context, singular, plural, count) { - var value = ngettext(context + '\x04' + singular, context + '\x04' + plural, count); - if (value.indexOf('\x04') != -1) { + var value = ngettext(context + '\\x04' + singular, context + '\\x04' + plural, count); + if (value.indexOf('\\x04') != -1) { value = ngettext(singular, plural, count); } return value; diff --git a/tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.mo b/tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.mo new file mode 100644 index 0000000000000000000000000000000000000000..34ba691029be7805899e6e28a03d36a5eb358d2f GIT binary patch literal 615 zcmYk3(P|Vi6o#X&D03m=g$Q~PF9b!hPG*IcX}1*HEv$BMSw?RqyK_35l1Y{%TkIRO z58#b&;DdPUBls--nNrpRAOHDh{!C6Xf1eC~Fw}kU0vv+}z<^X&;4!!ckHAmx9Q*jmWA~sF=riaPI)E;qi1n3xs+7ayesA_Fa2=_;)xbJu z!=??Qb|uedpKEdCNkyy>$0}0Ei(KX+FNA#0C4-k6uA_IBU4 z^w9bCAgiX;WP2z%%(Ek05Ls+whh=O6lb1{f|-o MyW5`FCmfai1AfY)s{jB1 literal 0 HcmV?d00001 diff --git a/tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.po b/tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.po new file mode 100644 index 0000000000..88cc35e88f --- /dev/null +++ b/tests/regressiontests/views/locale/de/LC_MESSAGES/djangojs.po @@ -0,0 +1,40 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: django tests\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2010-02-14 17:33+0100\n" +"PO-Revision-Date: 2011-01-21 21:37-0300\n" +"Last-Translator: Jannis Leidel \n" +"Language-Team: de \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1)\n" + +#: models.py:7 +msgctxt "month name" +msgid "May" +msgstr "Mai" + +#: models.py:9 +msgctxt "verb" +msgid "May" +msgstr "Kann" + +#: models.py:11 +msgid "%s item" +msgid_plural "%s items" +msgstr[0] "%s Element" +msgstr[1] "%s Elemente" + +#: models.py:11 +msgctxt "search" +msgid "%s result" +msgid_plural "%s results" +msgstr[0] "%s Resultat" +msgstr[1] "%s Resultate" diff --git a/tests/regressiontests/views/templates/jsi18n.html b/tests/regressiontests/views/templates/jsi18n.html new file mode 100644 index 0000000000..d6093c8ef4 --- /dev/null +++ b/tests/regressiontests/views/templates/jsi18n.html @@ -0,0 +1,44 @@ + + + + + + +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ + + diff --git a/tests/regressiontests/views/tests/__init__.py b/tests/regressiontests/views/tests/__init__.py index 12d0c59014..17f3f4562d 100644 --- a/tests/regressiontests/views/tests/__init__.py +++ b/tests/regressiontests/views/tests/__init__.py @@ -4,7 +4,7 @@ from .debug import (DebugViewTests, ExceptionReporterTests, ExceptionReporterTests, PlainTextReportTests, ExceptionReporterFilterTests, AjaxResponseExceptionReporterFilter) from .defaults import DefaultsTests -from .i18n import JsI18NTests, I18NTests, JsI18NTestsMultiPackage +from .i18n import JsI18NTests, I18NTests, JsI18NTestsMultiPackage, JavascriptI18nTests from .shortcuts import ShortcutTests from .specials import URLHandling from .static import StaticHelperTest, StaticUtilsTests, StaticTests diff --git a/tests/regressiontests/views/tests/i18n.py b/tests/regressiontests/views/tests/i18n.py index 601df6d512..d0bf7d2299 100644 --- a/tests/regressiontests/views/tests/i18n.py +++ b/tests/regressiontests/views/tests/i18n.py @@ -6,11 +6,17 @@ from os import path from django.conf import settings from django.core.urlresolvers import reverse -from django.test import TestCase -from django.utils import six -from django.utils.translation import override, get_language +from django.test import LiveServerTestCase, TestCase +from django.test.utils import override_settings +from django.utils import six, unittest +from django.utils.translation import override from django.utils.text import javascript_quote +try: + from selenium.webdriver.firefox import webdriver as firefox +except ImportError: + firefox = None + from ..urls import locale_dir @@ -152,3 +158,43 @@ class JsI18NTestsMultiPackage(TestCase): response = self.client.get('/views/jsi18n/') self.assertContains(response, javascript_quote('este texto de app3 debe ser traducido')) + + +@unittest.skipUnless(firefox, 'Selenium not installed') +class JavascriptI18nTests(LiveServerTestCase): + urls = 'regressiontests.views.urls' + + @classmethod + def setUpClass(cls): + cls.selenium = firefox.WebDriver() + super(JavascriptI18nTests, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + cls.selenium.quit() + super(JavascriptI18nTests, cls).tearDownClass() + + @override_settings(LANGUAGE_CODE='de') + def test_javascript_gettext(self): + extended_apps = list(settings.INSTALLED_APPS) + ['regressiontests.views'] + with self.settings(INSTALLED_APPS=extended_apps): + self.selenium.get('%s%s' % (self.live_server_url, '/jsi18n_template/')) + + elem = self.selenium.find_element_by_id("gettext") + self.assertEqual(elem.text, u"Entfernen") + elem = self.selenium.find_element_by_id("ngettext_sing") + self.assertEqual(elem.text, "1 Element") + elem = self.selenium.find_element_by_id("ngettext_plur") + self.assertEqual(elem.text, "455 Elemente") + elem = self.selenium.find_element_by_id("pgettext") + self.assertEqual(elem.text, "Kann") + elem = self.selenium.find_element_by_id("npgettext_sing") + self.assertEqual(elem.text, "1 Resultat") + elem = self.selenium.find_element_by_id("npgettext_plur") + self.assertEqual(elem.text, "455 Resultate") + + def test_escaping(self): + extended_apps = list(settings.INSTALLED_APPS) + ['regressiontests.views'] + with self.settings(INSTALLED_APPS=extended_apps): + response = self.client.get('%s%s' % (self.live_server_url, '/jsi18n_admin/')) + self.assertContains(response, '\\x04') diff --git a/tests/regressiontests/views/urls.py b/tests/regressiontests/views/urls.py index 90d2382f71..ae3b9c0a9e 100644 --- a/tests/regressiontests/views/urls.py +++ b/tests/regressiontests/views/urls.py @@ -32,6 +32,11 @@ js_info_dict_multi_packages2 = { 'packages': ('regressiontests.views.app3', 'regressiontests.views.app4'), } +js_info_dict_admin = { + 'domain': 'djangojs', + 'packages': ('django.contrib.admin', 'regressiontests.views'), +} + urlpatterns = patterns('', (r'^$', views.index_page), @@ -51,6 +56,8 @@ urlpatterns = patterns('', (r'^jsi18n_english_translation/$', 'django.views.i18n.javascript_catalog', js_info_dict_english_translation), (r'^jsi18n_multi_packages1/$', 'django.views.i18n.javascript_catalog', js_info_dict_multi_packages1), (r'^jsi18n_multi_packages2/$', 'django.views.i18n.javascript_catalog', js_info_dict_multi_packages2), + (r'^jsi18n_admin/$', 'django.views.i18n.javascript_catalog', js_info_dict_admin), + (r'^jsi18n_template/$', views.jsi18n), # Static views (r'^site_media/(?P.*)$', 'django.views.static.serve', {'document_root': media_dir}), diff --git a/tests/regressiontests/views/views.py b/tests/regressiontests/views/views.py index 2836d1bdde..ed9d61144a 100644 --- a/tests/regressiontests/views/views.py +++ b/tests/regressiontests/views/views.py @@ -51,6 +51,9 @@ def template_exception(request, n): return render_to_response('debug/template_exception.html', {'arg': except_args[int(n)]}) +def jsi18n(request): + return render_to_response('jsi18n.html') + # Some views to exercise the shortcuts def render_to_response_view(request): From f79013843d9e3ffced16eba34068fbff2cda2c0d Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Sat, 17 Nov 2012 15:59:34 +0100 Subject: [PATCH 143/302] Fixed typo introduced in 4a5e8087ac7676ef08e76275c1f756778b39c13e. --- tests/regressiontests/views/tests/i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regressiontests/views/tests/i18n.py b/tests/regressiontests/views/tests/i18n.py index d0bf7d2299..671becbbc9 100644 --- a/tests/regressiontests/views/tests/i18n.py +++ b/tests/regressiontests/views/tests/i18n.py @@ -181,7 +181,7 @@ class JavascriptI18nTests(LiveServerTestCase): self.selenium.get('%s%s' % (self.live_server_url, '/jsi18n_template/')) elem = self.selenium.find_element_by_id("gettext") - self.assertEqual(elem.text, u"Entfernen") + self.assertEqual(elem.text, "Entfernen") elem = self.selenium.find_element_by_id("ngettext_sing") self.assertEqual(elem.text, "1 Element") elem = self.selenium.find_element_by_id("ngettext_plur") From 7a38e86bde3cae12353fe0a4b5de9420ba780fb1 Mon Sep 17 00:00:00 2001 From: Preston Holmes Date: Sat, 17 Nov 2012 07:05:34 -0800 Subject: [PATCH 144/302] Fixed #19310 -- changed method docs formatting for custom file storage docs --- docs/howto/custom-file-storage.txt | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/howto/custom-file-storage.txt b/docs/howto/custom-file-storage.txt index 5f1dae17ef..51558d9715 100644 --- a/docs/howto/custom-file-storage.txt +++ b/docs/howto/custom-file-storage.txt @@ -27,9 +27,9 @@ You'll need to follow these steps: option = settings.CUSTOM_STORAGE_OPTIONS ... -#. Your storage class must implement the ``_open()`` and ``_save()`` methods, - along with any other methods appropriate to your storage class. See below for - more on these methods. +#. Your storage class must implement the :meth:`_open()` and :meth:`_save() + methods, along with any other methods appropriate to your storage class. See + below for more on these methods. In addition, if your class provides local file storage, it must override the ``path()`` method. @@ -46,8 +46,7 @@ Your custom storage system may override any of the storage methods explained in You'll also usually want to use hooks specifically designed for custom storage objects. These are: -``_open(name, mode='rb')`` -~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: _open(name, mode='rb') **Required**. @@ -56,8 +55,7 @@ uses to open the file. This must return a ``File`` object, though in most cases, you'll want to return some subclass here that implements logic specific to the backend storage system. -``_save(name, content)`` -~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: _save(name, content) Called by ``Storage.save()``. The ``name`` will already have gone through ``get_valid_name()`` and ``get_available_name()``, and the ``content`` will be a @@ -67,8 +65,8 @@ Should return the actual name of name of the file saved (usually the ``name`` passed in, but if the storage needs to change the file name return the new name instead). -``get_valid_name(name)`` -~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: get_valid_name(name) + Returns a filename suitable for use with the underlying storage system. The ``name`` argument passed to this method is the original filename sent to the @@ -78,8 +76,7 @@ how non-standard characters are converted to safe filenames. The code provided on ``Storage`` retains only alpha-numeric characters, periods and underscores from the original filename, removing everything else. -``get_available_name(name)`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: get_available_name(name) Returns a filename that is available in the storage mechanism, possibly taking the provided filename into account. The ``name`` argument passed to this method From 8c69278764ae31b0cd495d21b84444bf2471c103 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 17 Nov 2012 10:51:30 +0100 Subject: [PATCH 145/302] Fixed #18989 -- Removed unused condition in CursorWrapper Thanks zimnyx for the report. --- django/db/backends/util.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/django/db/backends/util.py b/django/db/backends/util.py index e029c42899..1ba23060e0 100644 --- a/django/db/backends/util.py +++ b/django/db/backends/util.py @@ -24,11 +24,9 @@ class CursorWrapper(object): self.db.set_dirty() def __getattr__(self, attr): - self.set_dirty() - if attr in self.__dict__: - return self.__dict__[attr] - else: - return getattr(self.cursor, attr) + if attr in ('execute', 'executemany', 'callproc'): + self.set_dirty() + return getattr(self.cursor, attr) def __iter__(self): return iter(self.cursor) From ec9d6b1122dd09168d8b2dbcabc745f22f0ef766 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 17 Nov 2012 17:06:24 +0100 Subject: [PATCH 146/302] Fixed #19226 -- Applied linebreaksbr to read-only fields in admin Thanks shadow for the report, and Melevir and thiderman for the patch. --- .../contrib/admin/templates/admin/includes/fieldset.html | 2 +- tests/regressiontests/admin_views/admin.py | 9 ++++++++- tests/regressiontests/admin_views/tests.py | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/django/contrib/admin/templates/admin/includes/fieldset.html b/django/contrib/admin/templates/admin/includes/fieldset.html index c8d08c880f..09bc971d2f 100644 --- a/django/contrib/admin/templates/admin/includes/fieldset.html +++ b/django/contrib/admin/templates/admin/includes/fieldset.html @@ -14,7 +14,7 @@ {% else %} {{ field.label_tag }} {% if field.is_readonly %} -

{{ field.contents }}

+

{{ field.contents|linebreaksbr }}

{% else %} {{ field.field }} {% endif %} diff --git a/tests/regressiontests/admin_views/admin.py b/tests/regressiontests/admin_views/admin.py index a5476e9eb7..6bb6ba59b0 100644 --- a/tests/regressiontests/admin_views/admin.py +++ b/tests/regressiontests/admin_views/admin.py @@ -388,7 +388,10 @@ class PrePopulatedPostAdmin(admin.ModelAdmin): class PostAdmin(admin.ModelAdmin): list_display = ['title', 'public'] - readonly_fields = ('posted', 'awesomeness_level', 'coolness', 'value', lambda obj: "foo") + readonly_fields = ( + 'posted', 'awesomeness_level', 'coolness', 'value', 'multiline', + lambda obj: "foo" + ) inlines = [ LinkInline @@ -402,6 +405,10 @@ class PostAdmin(admin.ModelAdmin): def value(self, instance): return 1000 + + def multiline(self, instance): + return "Multiline\ntest\nstring" + value.short_description = 'Value in $US' diff --git a/tests/regressiontests/admin_views/tests.py b/tests/regressiontests/admin_views/tests.py index 72dc6a3f97..b5e0f407e5 100644 --- a/tests/regressiontests/admin_views/tests.py +++ b/tests/regressiontests/admin_views/tests.py @@ -3149,6 +3149,10 @@ class ReadonlyTest(TestCase): self.assertContains(response, "Very awesome.") self.assertContains(response, "Unkown coolness.") self.assertContains(response, "foo") + + # Checks that multiline text in a readonly field gets
tags + self.assertContains(response, "Multiline
test
string") + self.assertContains(response, formats.localize(datetime.date.today() - datetime.timedelta(days=7)) ) From 9e7723f851dc6e4d5eea96ab808e1a4cdd4eabd7 Mon Sep 17 00:00:00 2001 From: Ramiro Morales Date: Fri, 9 Nov 2012 22:17:43 -0300 Subject: [PATCH 147/302] Refactor loaddata for readability. Thanks Claude Paroz and Daniel Moisset for review and feedback. --- django/core/management/commands/loaddata.py | 273 ++++++++++---------- 1 file changed, 140 insertions(+), 133 deletions(-) diff --git a/django/core/management/commands/loaddata.py b/django/core/management/commands/loaddata.py index 32ae8abf5a..f6f1b1039a 100644 --- a/django/core/management/commands/loaddata.py +++ b/django/core/management/commands/loaddata.py @@ -1,11 +1,9 @@ from __future__ import unicode_literals -import sys import os import gzip import zipfile from optparse import make_option -import traceback from django.conf import settings from django.core import serializers @@ -39,10 +37,10 @@ class Command(BaseCommand): def handle(self, *fixture_labels, **options): - ignore = options.get('ignore') - using = options.get('database') + self.ignore = options.get('ignore') + self.using = options.get('database') - connection = connections[using] + connection = connections[self.using] if not len(fixture_labels): raise CommandError( @@ -50,8 +48,7 @@ class Command(BaseCommand): "least one fixture in the command line." ) - verbosity = int(options.get('verbosity')) - show_traceback = options.get('traceback') + self.verbosity = int(options.get('verbosity')) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking @@ -62,12 +59,10 @@ class Command(BaseCommand): commit = options.get('commit', True) # Keep a count of the installed objects and fixtures - fixture_count = 0 - loaded_object_count = 0 - fixture_object_count = 0 - models = set() - - humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' + self.fixture_count = 0 + self.loaded_object_count = 0 + self.fixture_object_count = 0 + self.models = set() # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if @@ -77,9 +72,9 @@ class Command(BaseCommand): # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: - transaction.commit_unless_managed(using=using) - transaction.enter_transaction_management(using=using) - transaction.managed(True, using=using) + transaction.commit_unless_managed(using=self.using) + transaction.enter_transaction_management(using=self.using) + transaction.managed(True, using=self.using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): @@ -89,13 +84,13 @@ class Command(BaseCommand): def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) - compression_types = { + self.compression_types = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: - compression_types['bz2'] = bz2.BZ2File + self.compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): @@ -112,113 +107,11 @@ class Command(BaseCommand): try: with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: - parts = fixture_label.split('.') - - if len(parts) > 1 and parts[-1] in compression_types: - compression_formats = [parts[-1]] - parts = parts[:-1] - else: - compression_formats = compression_types.keys() - - if len(parts) == 1: - fixture_name = parts[0] - formats = serializers.get_public_serializer_formats() - else: - fixture_name, format = '.'.join(parts[:-1]), parts[-1] - if format in serializers.get_public_serializer_formats(): - formats = [format] - else: - formats = [] - - if formats: - if verbosity >= 2: - self.stdout.write("Loading '%s' fixtures..." % fixture_name) - else: - raise CommandError( - "Problem installing fixture '%s': %s is not a known serialization format." % - (fixture_name, format)) - - if os.path.isabs(fixture_name): - fixture_dirs = [fixture_name] - else: - fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] - - for fixture_dir in fixture_dirs: - if verbosity >= 2: - self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) - - label_found = False - for combo in product([using, None], formats, compression_formats): - database, format, compression_format = combo - file_name = '.'.join( - p for p in [ - fixture_name, database, format, compression_format - ] - if p - ) - - if verbosity >= 3: - self.stdout.write("Trying %s for %s fixture '%s'..." % \ - (humanize(fixture_dir), file_name, fixture_name)) - full_path = os.path.join(fixture_dir, file_name) - open_method = compression_types[compression_format] - try: - fixture = open_method(full_path, 'r') - except IOError: - if verbosity >= 2: - self.stdout.write("No %s fixture '%s' in %s." % \ - (format, fixture_name, humanize(fixture_dir))) - else: - try: - if label_found: - raise CommandError("Multiple fixtures named '%s' in %s. Aborting." % - (fixture_name, humanize(fixture_dir))) - - fixture_count += 1 - objects_in_fixture = 0 - loaded_objects_in_fixture = 0 - if verbosity >= 2: - self.stdout.write("Installing %s fixture '%s' from %s." % \ - (format, fixture_name, humanize(fixture_dir))) - - objects = serializers.deserialize(format, fixture, using=using, ignorenonexistent=ignore) - - for obj in objects: - objects_in_fixture += 1 - if router.allow_syncdb(using, obj.object.__class__): - loaded_objects_in_fixture += 1 - models.add(obj.object.__class__) - try: - obj.save(using=using) - except (DatabaseError, IntegrityError) as e: - e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { - 'app_label': obj.object._meta.app_label, - 'object_name': obj.object._meta.object_name, - 'pk': obj.object.pk, - 'error_msg': force_text(e) - },) - raise - - loaded_object_count += loaded_objects_in_fixture - fixture_object_count += objects_in_fixture - label_found = True - except Exception as e: - if not isinstance(e, CommandError): - e.args = ("Problem installing fixture '%s': %s" % (full_path, e),) - raise - finally: - fixture.close() - - # If the fixture we loaded contains 0 objects, assume that an - # error was encountered during fixture loading. - if objects_in_fixture == 0: - raise CommandError( - "No fixture data found for '%s'. (File format may be invalid.)" % - (fixture_name)) + self.load_label(fixture_label, app_fixtures) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added - table_names = [model._meta.db_table for model in models] + table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: @@ -229,31 +122,31 @@ class Command(BaseCommand): raise except Exception as e: if commit: - transaction.rollback(using=using) - transaction.leave_transaction_management(using=using) + transaction.rollback(using=self.using) + transaction.leave_transaction_management(using=self.using) raise # If we found even one object in a fixture, we need to reset the # database sequences. - if loaded_object_count > 0: - sequence_sql = connection.ops.sequence_reset_sql(no_style(), models) + if self.loaded_object_count > 0: + sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models) if sequence_sql: - if verbosity >= 2: + if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") for line in sequence_sql: cursor.execute(line) if commit: - transaction.commit(using=using) - transaction.leave_transaction_management(using=using) + transaction.commit(using=self.using) + transaction.leave_transaction_management(using=self.using) - if verbosity >= 1: - if fixture_object_count == loaded_object_count: + if self.verbosity >= 1: + if self.fixture_object_count == self.loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % ( - loaded_object_count, fixture_count)) + self.loaded_object_count, self.fixture_count)) else: self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % ( - loaded_object_count, fixture_object_count, fixture_count)) + self.loaded_object_count, self.fixture_object_count, self.fixture_count)) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to @@ -261,3 +154,117 @@ class Command(BaseCommand): # incorrect results. See Django #7572, MySQL #37735. if commit: connection.close() + + def load_label(self, fixture_label, app_fixtures): + + parts = fixture_label.split('.') + + if len(parts) > 1 and parts[-1] in self.compression_types: + compression_formats = [parts[-1]] + parts = parts[:-1] + else: + compression_formats = self.compression_types.keys() + + if len(parts) == 1: + fixture_name = parts[0] + formats = serializers.get_public_serializer_formats() + else: + fixture_name, format = '.'.join(parts[:-1]), parts[-1] + if format in serializers.get_public_serializer_formats(): + formats = [format] + else: + formats = [] + + if formats: + if self.verbosity >= 2: + self.stdout.write("Loading '%s' fixtures..." % fixture_name) + else: + raise CommandError( + "Problem installing fixture '%s': %s is not a known serialization format." % + (fixture_name, format)) + + if os.path.isabs(fixture_name): + fixture_dirs = [fixture_name] + else: + fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] + + for fixture_dir in fixture_dirs: + self.process_dir(fixture_dir, fixture_name, compression_formats, + formats) + + def process_dir(self, fixture_dir, fixture_name, compression_formats, + serialization_formats): + + humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' + + if self.verbosity >= 2: + self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) + + label_found = False + for combo in product([self.using, None], serialization_formats, compression_formats): + database, format, compression_format = combo + file_name = '.'.join( + p for p in [ + fixture_name, database, format, compression_format + ] + if p + ) + + if self.verbosity >= 3: + self.stdout.write("Trying %s for %s fixture '%s'..." % \ + (humanize(fixture_dir), file_name, fixture_name)) + full_path = os.path.join(fixture_dir, file_name) + open_method = self.compression_types[compression_format] + try: + fixture = open_method(full_path, 'r') + except IOError: + if self.verbosity >= 2: + self.stdout.write("No %s fixture '%s' in %s." % \ + (format, fixture_name, humanize(fixture_dir))) + else: + try: + if label_found: + raise CommandError("Multiple fixtures named '%s' in %s. Aborting." % + (fixture_name, humanize(fixture_dir))) + + self.fixture_count += 1 + objects_in_fixture = 0 + loaded_objects_in_fixture = 0 + if self.verbosity >= 2: + self.stdout.write("Installing %s fixture '%s' from %s." % \ + (format, fixture_name, humanize(fixture_dir))) + + objects = serializers.deserialize(format, fixture, using=self.using, ignorenonexistent=self.ignore) + + for obj in objects: + objects_in_fixture += 1 + if router.allow_syncdb(self.using, obj.object.__class__): + loaded_objects_in_fixture += 1 + self.models.add(obj.object.__class__) + try: + obj.save(using=self.using) + except (DatabaseError, IntegrityError) as e: + e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % { + 'app_label': obj.object._meta.app_label, + 'object_name': obj.object._meta.object_name, + 'pk': obj.object.pk, + 'error_msg': force_text(e) + },) + raise + + self.loaded_object_count += loaded_objects_in_fixture + self.fixture_object_count += objects_in_fixture + label_found = True + except Exception as e: + if not isinstance(e, CommandError): + e.args = ("Problem installing fixture '%s': %s" % (full_path, e),) + raise + finally: + fixture.close() + + # If the fixture we loaded contains 0 objects, assume that an + # error was encountered during fixture loading. + if objects_in_fixture == 0: + raise CommandError( + "No fixture data found for '%s'. (File format may be invalid.)" % + (fixture_name)) From 2a67374b51c5705d5c64a5d7117ad8552e98b4bb Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Thu, 11 Oct 2012 23:20:25 +0200 Subject: [PATCH 148/302] Fixed #19036 -- Fixed base64 uploads decoding Thanks anthony at adsorbtion.org for the report, and johannesl for bringing the patch up-to-date. --- django/http/multipartparser.py | 6 ++++++ tests/regressiontests/file_uploads/tests.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py index 9413a1eabb..edf98f6e49 100644 --- a/django/http/multipartparser.py +++ b/django/http/multipartparser.py @@ -199,6 +199,12 @@ class MultiPartParser(object): for chunk in field_stream: if transfer_encoding == 'base64': # We only special-case base64 transfer encoding + # We should always read base64 streams by multiple of 4 + over_bytes = len(chunk) % 4 + if over_bytes: + over_chunk = field_stream.read(4 - over_bytes) + chunk += over_chunk + try: chunk = base64.b64decode(chunk) except Exception as e: diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py index 8fa140bc18..45e7342abd 100644 --- a/tests/regressiontests/file_uploads/tests.py +++ b/tests/regressiontests/file_uploads/tests.py @@ -74,15 +74,14 @@ class FileUploadTests(TestCase): self.assertEqual(response.status_code, 200) - def test_base64_upload(self): - test_string = "This data will be transmitted base64-encoded." + def _test_base64_upload(self, content): payload = client.FakePayload("\r\n".join([ '--' + client.BOUNDARY, 'Content-Disposition: form-data; name="file"; filename="test.txt"', 'Content-Type: application/octet-stream', 'Content-Transfer-Encoding: base64', '',])) - payload.write(b"\r\n" + base64.b64encode(force_bytes(test_string)) + b"\r\n") + payload.write(b"\r\n" + base64.b64encode(force_bytes(content)) + b"\r\n") payload.write('--' + client.BOUNDARY + '--\r\n') r = { 'CONTENT_LENGTH': len(payload), @@ -94,7 +93,13 @@ class FileUploadTests(TestCase): response = self.client.request(**r) received = json.loads(response.content.decode('utf-8')) - self.assertEqual(received['file'], test_string) + self.assertEqual(received['file'], content) + + def test_base64_upload(self): + self._test_base64_upload("This data will be transmitted base64-encoded.") + + def test_big_base64_upload(self): + self._test_base64_upload("Big data" * 68000) # > 512Kb def test_unicode_file_name(self): tdir = sys_tempfile.mkdtemp() From e0363c688d9b85124ec4fbac00debe5f4dc1e63c Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Sat, 17 Nov 2012 19:16:30 +0100 Subject: [PATCH 149/302] Fixed #19114 -- Fixed LogEntry unicode representation Thanks niko at neagee.net for the report and Emil Stenstrom for the patch. --- django/contrib/admin/models.py | 13 ++++++++----- tests/regressiontests/admin_util/tests.py | 4 ++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/django/contrib/admin/models.py b/django/contrib/admin/models.py index e1d3b40d01..b697d7bdc8 100644 --- a/django/contrib/admin/models.py +++ b/django/contrib/admin/models.py @@ -4,7 +4,7 @@ from django.db import models from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.contrib.admin.util import quote -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import ugettext, ugettext_lazy as _ from django.utils.encoding import smart_text from django.utils.encoding import python_2_unicode_compatible @@ -42,13 +42,16 @@ class LogEntry(models.Model): def __str__(self): if self.action_flag == ADDITION: - return _('Added "%(object)s".') % {'object': self.object_repr} + return ugettext('Added "%(object)s".') % {'object': self.object_repr} elif self.action_flag == CHANGE: - return _('Changed "%(object)s" - %(changes)s') % {'object': self.object_repr, 'changes': self.change_message} + return ugettext('Changed "%(object)s" - %(changes)s') % { + 'object': self.object_repr, + 'changes': self.change_message, + } elif self.action_flag == DELETION: - return _('Deleted "%(object)s."') % {'object': self.object_repr} + return ugettext('Deleted "%(object)s."') % {'object': self.object_repr} - return _('LogEntry Object') + return ugettext('LogEntry Object') def is_addition(self): return self.action_flag == ADDITION diff --git a/tests/regressiontests/admin_util/tests.py b/tests/regressiontests/admin_util/tests.py index ef8a91d1db..e9e122a9f0 100644 --- a/tests/regressiontests/admin_util/tests.py +++ b/tests/regressiontests/admin_util/tests.py @@ -274,6 +274,10 @@ class UtilTests(unittest.TestCase): six.text_type(log_entry).startswith('Deleted ') ) + # Make sure custom action_flags works + log_entry.action_flag = 4 + self.assertEqual(six.text_type(log_entry), 'LogEntry Object') + def test_safestring_in_field_label(self): # safestring should not be escaped class MyForm(forms.Form): From 7058b595b668b49daef8beb76a2b5c5f1d991b00 Mon Sep 17 00:00:00 2001 From: Tim Graham Date: Tue, 30 Oct 2012 20:09:49 -0400 Subject: [PATCH 150/302] Fixed #16779 - Added a contributing tutorial Thank-you Taavi Taijala for the draft patch! --- docs/index.txt | 3 +- .../contributing/writing-code/unit-tests.txt | 4 + docs/intro/contributing.txt | 580 ++++++++++++++++++ docs/intro/index.txt | 1 + 4 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 docs/intro/contributing.txt diff --git a/docs/index.txt b/docs/index.txt index a6d9ed2b13..e6eb77c98f 100644 --- a/docs/index.txt +++ b/docs/index.txt @@ -47,7 +47,8 @@ Are you new to Django or to programming? This is the place to start! :doc:`Part 4 ` * **Advanced Tutorials:** - :doc:`How to write reusable apps ` + :doc:`How to write reusable apps ` | + :doc:`Writing your first patch for Django ` The model layer =============== diff --git a/docs/internals/contributing/writing-code/unit-tests.txt b/docs/internals/contributing/writing-code/unit-tests.txt index a828b06b36..71666a169e 100644 --- a/docs/internals/contributing/writing-code/unit-tests.txt +++ b/docs/internals/contributing/writing-code/unit-tests.txt @@ -38,6 +38,8 @@ with this sample ``settings`` module, ``cd`` into the Django If you get an ``ImportError: No module named django.contrib`` error, you need to add your install of Django to your ``PYTHONPATH``. +.. _running-unit-tests-settings: + Using another ``settings`` module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -133,6 +135,8 @@ Then, run the tests normally, for example: ./runtests.py --settings=test_sqlite admin_inlines +.. _running-unit-tests-dependencies: + Running all the tests ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/intro/contributing.txt b/docs/intro/contributing.txt new file mode 100644 index 0000000000..a343814c02 --- /dev/null +++ b/docs/intro/contributing.txt @@ -0,0 +1,580 @@ +=================================== +Writing your first patch for Django +=================================== + +Introduction +============ + +Interested in giving back to the community a little? Maybe you've found a bug +in Django that you'd like to see fixed, or maybe there's a small feature you +want added. + +Contributing back to Django itself is the best way to see your own concerns +addressed. This may seem daunting at first, but it's really pretty simple. +We'll walk you through the entire process, so you can learn by example. + +Who's this tutorial for? +------------------------ + +For this tutorial, we expect that you have at least a basic understanding of +how Django works. This means you should be comfortable going through the +existing tutorials on :doc:`writing your first Django app`. +In addition, you should have a good understanding of Python itself. But if you +don't, `Dive Into Python`__ is a fantastic (and free) online book for beginning +Python programmers. + +Those of you who are unfamiliar with version control systems and Trac will find +that this tutorial and its links include just enough information to get started. +However, you'll probably want to read some more about these different tools if +you plan on contributing to Django regularly. + +For the most part though, this tutorial tries to explain as much as possible, +so that it can be of use to the widest audience. + +.. admonition:: Where to get help: + + If you're having trouble going through this tutorial, please post a message + to `django-developers`__ or drop by `#django-dev on irc.freenode.net`__ to + chat with other Django users who might be able to help. + +__ http://diveintopython.net/toc/index.html +__ http://groups.google.com/group/django-developers +__ irc://irc.freenode.net/django-dev + +What does this tutorial cover? +------------------------------ + +We'll be walking you through contributing a patch to Django for the first time. +By the end of this tutorial, you should have a basic understanding of both the +tools and the processes involved. Specifically, we'll be covering the following: + +* Installing Git. +* How to download a development copy of Django. +* Running Django's test suite. +* Writing a test for your patch. +* Writing the code for your patch. +* Testing your patch. +* Generating a patch file for your changes. +* Where to look for more information. + +Once you're done with the tutorial, you can look through the rest of +:doc:`Django's documentation on contributing`. +It contains lots of great information and is a must read for anyone who'd like +to become a regular contributor to Django. If you've got questions, it's +probably got the answers. + +Installing Git +============== + +For this tutorial, you'll need Git installed to download the current +development version of Django and to generate patch files for the changes you +make. + +To check whether or not you have Git installed, enter ``git`` into the command +line. If you get messages saying that this command could be found, you'll have +to download and install it, see `Git's download page`__. + +If you're not that familiar with Git, you can always find out more about its +commands (once it's installed) by typing ``git help`` into the command line. + +__ http://git-scm.com/download + +Getting a copy of Django's development version +============================================== + +The first step to contributing to Django is to get a copy of the source code. +From the command line, use the ``cd`` command to navigate to the directory +where you'll want your local copy of Django to live. + +Download the Django source code repository using the following command:: + + git clone https://github.com/django/django.git + +.. note:: + + For users who wish to use `virtualenv`__, you can use:: + + pip install -e /path/to/your/local/clone/django/ + + to link your cloned checkout into a virtual environment. This is a great + option to isolate your development copy of Django from the rest of your + system and avoids potential package conflicts. + +__ http://www.virtualenv.org + +Rolling back to a previous revision of Django +============================================= + +For this tutorial, we'll be using `ticket #17549`__ as a case study, so we'll +rewind Django's version history in git to before that ticket's patch was +applied. This will allow us to go through all of the steps involved in writing +that patch from scratch, including running Django's test suite. + +**Keep in mind that while we'll be using an older revision of Django's trunk +for the purposes of the tutorial below, you should always use the current +development revision of Django when working on your own patch for a ticket!** + +.. note:: + + The patch for this ticket was written by Ulrich Petri, and it was applied + to Django as `commit ac2052ebc84c45709ab5f0f25e685bf656ce79bc`__. + Consequently, we'll be using the revision of Django just prior to that, + `commit 39f5bc7fc3a4bb43ed8a1358b17fe0521a1a63ac`__. + +__ https://code.djangoproject.com/ticket/17549 +__ https://github.com/django/django/commit/ac2052ebc84c45709ab5f0f25e685bf656ce79bc +__ https://github.com/django/django/commit/39f5bc7fc3a4bb43ed8a1358b17fe0521a1a63ac + +Navigate into Django's root directory (that's the one that contains ``django``, +``docs``, ``tests``, ``AUTHORS``, etc.). You can then check out the older +revision of Django that we'll be using in the tutorial below:: + + git checkout 39f5bc7fc3a4bb43ed8a1358b17fe0521a1a63ac + +Running Django's test suite for the first time +============================================== + +When contributing to Django it's very important that your code changes don't +introduce bugs into other areas of Django. One way to check that Django still +works after you make your changes is by running Django's test suite. If all +the tests still pass, then you can be reasonably sure that your changes +haven't completely broken Django. If you've never run Django's test suite +before, it's a good idea to run it once beforehand just to get familiar with +what its output is supposed to look like. + +We can run the test suite by simply ``cd``-ing into the Django ``tests/`` +directory and, if you're using GNU/Linux, Mac OS X or some other flavor of +Unix, run:: + + PYTHONPATH=.. python runtests.py --settings=test_sqlite + +If you're on Windows, the above should work provided that you are using +"Git Bash" provided by the default Git install. GitHub has a `nice tutorial`__. + +__ https://help.github.com/articles/set-up-git#platform-windows + +.. note:: + + If you're using ``virtualenv``, you can omit ``PYTHONPATH=..`` when running + the tests. This instructs Python to look for Django in the parent directory + of ``tests``. ``virtualenv`` puts your copy of Django on the ``PYTHONPATH`` + automatically. + +Now sit back and relax. Django's entire test suite has over 4800 different +tests, so it can take anywhere from 5 to 15 minutes to run, depending on the +speed of your computer. + +While Django's test suite is running, you'll see a stream of characters +representing the status of each test as it's run. ``E`` indicates that an error +was raised during a test, and ``F`` indicates that a test's assertions failed. +Both of these are considered to be test failures. Meanwhile, ``x`` and ``s`` +indicated expected failures and skipped tests, respectively. Dots indicate +passing tests. + +Skipped tests are typically due to missing external libraries required to run +the test; see :ref:`running-unit-tests-dependencies` for a list of dependencies +and be sure to install any for tests related to the changes you are making (we +won't need any for this tutorial). + +Once the tests complete, you should be greeted with a message informing you +whether the test suite passed or failed. Since you haven't yet made any changes +to Django's code, the entire test suite **should** pass. If you get failures or +errors make sure you've followed all of the previous steps properly. See +:ref:`running-unit-tests` for more information. + +Note that the latest Django trunk may not always be stable. When developing +against trunk, you can check `Django's continuous integration builds`__ to +determine if the failures are specific to your machine or if they are also +present in Django's official builds. If you click to view a particular build, +you can view the "Configuration Matrix" which shows failures broken down by +Python version and database backend. + +__ http://ci.djangoproject.com/ + +.. note:: + + For this tutorial and the ticket we're working on, testing against SQLite + is sufficient, however, it's possible (and sometimes necessary) to + :ref:`run the tests using a different database + `. + +Writing some tests for your ticket +================================== + +In most cases, for a patch to be accepted into Django it has to include tests. +For bug fix patches, this means writing a regression test to ensure that the +bug is never reintroduced into Django later on. A regression test should be +written in such a way that it will fail while the bug still exists and pass +once the bug has been fixed. For patches containing new features, you'll need +to include tests which ensure that the new features are working correctly. +They too should fail when the new feature is not present, and then pass once it +has been implemented. + +A good way to do this is to write your new tests first, before making any +changes to the code. This style of development is called +`test-driven development`__ and can be applied to both entire projects and +single patches. After writing your tests, you then run them to make sure that +they do indeed fail (since you haven't fixed that bug or added that feature +yet). If your new tests don't fail, you'll need to fix them so that they do. +After all, a regression test that passes regardless of whether a bug is present +is not very helpful at preventing that bug from reoccurring down the road. + +Now for our hands-on example. + +__ http://en.wikipedia.org/wiki/Test-driven_development + +Writing some tests for ticket #17549 +------------------------------------ + +`Ticket #17549`__ describes the following, small feature addition: + + It's useful for URLField to give you a way to open the URL; otherwise you + might as well use a CharField. + +In order to resolve this ticket, we'll add a ``render`` method to the +``AdminURLFieldWidget`` in order to display a clickable link above the input +widget. Before we make those changes though, we're going to write a couple +tests to verify that our modification functions correctly and continues to +function correctly in the future. + +Navigate to Django's ``tests/regressiontests/admin_widgets/`` folder and +open the ``tests.py`` file. Add the following code on line 269 right before the +``AdminFileWidgetTest`` class:: + + class AdminURLWidgetTest(DjangoTestCase): + def test_render(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', '')), + '' + ) + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example.com')), + '

Currently:http://example.com
Change:

' + ) + + def test_render_idn(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example-äüö.com')), + '

Currently:http://example-äüö.com
Change:

' + ) + + def test_render_quoting(self): + w = widgets.AdminURLFieldWidget() + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example.com/some text')), + '

Currently:http://example.com/<sometag>some text</sometag>
Change:

' + ) + self.assertHTMLEqual( + conditional_escape(w.render('test', 'http://example-äüö.com/some text')), + '

Currently:http://example-äüö.com/<sometag>some text</sometag>
Change:

' + ) + +The new tests check to see that the ``render`` method we'll be adding works +correctly in a couple different situations. + +.. admonition:: But this testing thing looks kinda hard... + + If you've never had to deal with tests before, they can look a little hard + to write at first glance. Fortunately, testing is a *very* big subject in + computer programming, so there's lots of information out there: + + * A good first look at writing tests for Django can be found in the + documentation on :doc:`Testing Django applications`. + * Dive Into Python (a free online book for beginning Python developers) + includes a great `introduction to Unit Testing`__. + * After reading those, if you want something a little meatier to sink + your teeth into, there's always the `Python unittest documentation`__. + +__ https://code.djangoproject.com/ticket/17549 +__ http://diveintopython.net/unit_testing/index.html +__ http://docs.python.org/library/unittest.html + +Running your new test +--------------------- + +Remember that we haven't actually made any modifications to +``AdminURLFieldWidget`` yet, so our tests are going to fail. Let's run all the +tests in the ``model_forms_regress`` folder to make sure that's really what +happens. From the command line, ``cd`` into the Django ``tests/`` directory +and run:: + + PYTHONPATH=.. python runtests.py --settings=test_sqlite admin_widgets + +If the tests ran correctly, you should see three failures corresponding to each +of the test methods we added. If all of the tests passed, then you'll want to +make sure that you added the new test shown above to the appropriate folder and +class. + +Writing the code for your ticket +================================ + +Next we'll be adding the functionality described in `ticket #17549`__ to Django. + +Writing the code for ticket #17549 +---------------------------------- + +Navigate to the ``django/django/contrib/admin/`` folder and open the +``widgets.py`` file. Find the ``AdminURLFieldWidget`` class on line 302 and add +the following ``render`` method after the existing ``__init__`` method:: + + def render(self, name, value, attrs=None): + html = super(AdminURLFieldWidget, self).render(name, value, attrs) + if value: + value = force_text(self._format_value(value)) + final_attrs = {'href': mark_safe(smart_urlquote(value))} + html = format_html( + '

{0} {2}
{3} {4}

', + _('Currently:'), flatatt(final_attrs), value, + _('Change:'), html + ) + return html + +Verifying your test now passes +------------------------------ + +Once you're done modifying Django, we need to make sure that the tests we wrote +earlier pass, so we can see whether the code we wrote above is working +correctly. To run the tests in the ``admin_widgets`` folder, ``cd`` into the +Django ``tests/`` directory and run:: + + PYTHONPATH=.. python runtests.py --settings=test_sqlite admin_widgets + +Oops, good thing we wrote those tests! You should still see 3 failures with +the following exception:: + + NameError: global name 'smart_urlquote' is not defined + +We forgot to add the import for that method. Go ahead and add the +``smart_urlquote`` import at the end of line 13 of +``django/contrib/admin/widgets.py`` so it looks as follows:: + + from django.utils.html import escape, format_html, format_html_join, smart_urlquote + +Re-run the tests and everything should pass. If it doesn't, make sure you +correctly modified the ``AdminURLFieldWidget`` class as shown above and +copied the new tests correctly. + +__ https://code.djangoproject.com/ticket/17549 + +Running Django's test suite for the second time +=============================================== + +Once you've verified that your patch and your test are working correctly, it's +a good idea to run the entire Django test suite just to verify that your change +hasn't introduced any bugs into other areas of Django. While successfully +passing the entire test suite doesn't guarantee your code is bug free, it does +help identify many bugs and regressions that might otherwise go unnoticed. + +To run the entire Django test suite, ``cd`` into the Django ``tests/`` +directory and run:: + + PYTHONPATH=.. python runtests.py --settings=test_sqlite + +As long as you don't see any failures, you're good to go. Note that this fix +also made a `small CSS change`__ to format the new widget. You can make the +change if you'd like, but we'll skip it for now in the interest of brevity. + +__ https://github.com/django/django/commit/ac2052ebc84c45709ab5f0f25e685bf656ce79bc#diff-0 + +Writing Documentation +===================== + +This is a new feature, so it should be documented. Add the following on line +925 of ``django/docs/ref/models/fields.txt`` beneath the existing docs for +``URLField``:: + + .. versionadded:: 1.5 + + The current value of the field will be displayed as a clickable link above the + input widget. + +For more information on writing documentation, including an explanation of what +the ``versionadded`` bit is all about, see +:doc:`/internals/contributing/writing-documentation`. That page also includes +an explanation of how to build a copy of the documentation locally, so you can +preview the HTML that will be generated. + +Generating a patch for your changes +=================================== + +Now it's time to generate a patch file that can be uploaded to Trac or applied +to another copy of Django. To get a look at the content of your patch, run the +following command:: + + git diff + +This will display the differences between your current copy of Django (with +your changes) and the revision that you initially checked out earlier in the +tutorial. + +Once you're done looking at the patch, hit the ``q`` key to exit back to the +command line. If the patch's content looked okay, you can run the following +command to save the patch file to your current working directory:: + + git diff > 17549.diff + +You should now have a file in the root Django directory called ``17549.diff``. +This patch file contains all your changes and should look this: + +.. code-block:: diff + + diff --git a/django/contrib/admin/widgets.py b/django/contrib/admin/widgets.py + index 1e0bc2d..9e43a10 100644 + --- a/django/contrib/admin/widgets.py + +++ b/django/contrib/admin/widgets.py + @@ -10,7 +10,7 @@ from django.contrib.admin.templatetags.admin_static import static + from django.core.urlresolvers import reverse + from django.forms.widgets import RadioFieldRenderer + from django.forms.util import flatatt + -from django.utils.html import escape, format_html, format_html_join + +from django.utils.html import escape, format_html, format_html_join, smart_urlquote + from django.utils.text import Truncator + from django.utils.translation import ugettext as _ + from django.utils.safestring import mark_safe + @@ -306,6 +306,18 @@ class AdminURLFieldWidget(forms.TextInput): + final_attrs.update(attrs) + super(AdminURLFieldWidget, self).__init__(attrs=final_attrs) + + + def render(self, name, value, attrs=None): + + html = super(AdminURLFieldWidget, self).render(name, value, attrs) + + if value: + + value = force_text(self._format_value(value)) + + final_attrs = {'href': mark_safe(smart_urlquote(value))} + + html = format_html( + + '

{0} {2}
{3} {4}

', + + _('Currently:'), flatatt(final_attrs), value, + + _('Change:'), html + + ) + + return html + + + class AdminIntegerFieldWidget(forms.TextInput): + class_name = 'vIntegerField' + + diff --git a/docs/ref/models/fields.txt b/docs/ref/models/fields.txt + index 809d56e..d44f85f 100644 + --- a/docs/ref/models/fields.txt + +++ b/docs/ref/models/fields.txt + @@ -922,6 +922,10 @@ Like all :class:`CharField` subclasses, :class:`URLField` takes the optional + :attr:`~CharField.max_length`argument. If you don't specify + :attr:`~CharField.max_length`, a default of 200 is used. + + +.. versionadded:: 1.5 + + + +The current value of the field will be displayed as a clickable link above the + +input widget. + + Relationship fields + =================== + diff --git a/tests/regressiontests/admin_widgets/tests.py b/tests/regressiontests/admin_widgets/tests.py + index 4b11543..94acc6d 100644 + --- a/tests/regressiontests/admin_widgets/tests.py + +++ b/tests/regressiontests/admin_widgets/tests.py + @@ -265,6 +265,35 @@ class AdminSplitDateTimeWidgetTest(DjangoTestCase): + '

Datum:
Zeit:

', + ) + + +class AdminURLWidgetTest(DjangoTestCase): + + def test_render(self): + + w = widgets.AdminURLFieldWidget() + + self.assertHTMLEqual( + + conditional_escape(w.render('test', '')), + + '' + + ) + + self.assertHTMLEqual( + + conditional_escape(w.render('test', 'http://example.com')), + + '

Currently:http://example.com
Change:

' + + ) + + + + def test_render_idn(self): + + w = widgets.AdminURLFieldWidget() + + self.assertHTMLEqual( + + conditional_escape(w.render('test', 'http://example-äüö.com')), + + '

Currently:http://example-äüö.com
Change:

' + + ) + + + + def test_render_quoting(self): + + w = widgets.AdminURLFieldWidget() + + self.assertHTMLEqual( + + conditional_escape(w.render('test', 'http://example.com/some text')), + + '

Currently:http://example.com/<sometag>some text</sometag>
Change:

' + + ) + + self.assertHTMLEqual( + + conditional_escape(w.render('test', 'http://example-äüö.com/some text')), + + '

Currently:http://example-äüö.com/<sometag>some text</sometag>
Change:

' + + ) + + class AdminFileWidgetTest(DjangoTestCase): + def test_render(self): + +So what do I do next? +===================== + +Congratulations, you've generated your very first Django patch! Now that you've +got that under your belt, you can put those skills to good use by helping to +improve Django's codebase. Generating patches and attaching them to Trac +tickets is useful, however, since we are using git - adopting a more :doc:`git +oriented workflow ` is +recommended. + +Since we never committed our changes locally, perform the following to get your +git branch back to a good starting point:: + + git reset --hard HEAD + git checkout master + +More information for new contributors +------------------------------------- + +Before you get too into writing patches for Django, there's a little more +information on contributing that you should probably take a look at: + +* You should make sure to read Django's documentation on + :doc:`claiming tickets and submitting patches + `. + It covers Trac etiquette, how to claim tickets for yourself, expected + coding style for patches, and many other important details. +* First time contributors should also read Django's :doc:`documentation + for first time contributors`. + It has lots of good advice for those of us who are new to helping out + with Django. +* After those, if you're still hungry for more information about + contributing, you can always browse through the rest of + :doc:`Django's documentation on contributing`. + It contains a ton of useful information and should be your first source + for answering any questions you might have. + +Finding your first real ticket +------------------------------ + +Once you've looked through some of that information, you'll be ready to go out +and find a ticket of your own to write a patch for. Pay special attention to +tickets with the "easy pickings" criterion. These tickets are often much +simpler in nature and are great for first time contributors. Once you're +familiar with contributing to Django, you can move on to writing patches for +more difficult and complicated tickets. + +If you just want to get started already (and nobody would blame you!), try +taking a look at the list of `easy tickets that need patches`__ and the +`easy tickets that have patches which need improvement`__. If you're familiar +with writing tests, you can also look at the list of +`easy tickets that need tests`__. Just remember to follow the guidelines about +claiming tickets that were mentioned in the link to Django's documentation on +:doc:`claiming tickets and submitting patches +`. + +__ https://code.djangoproject.com/query?status=new&status=reopened&has_patch=0&easy=1&col=id&col=summary&col=status&col=owner&col=type&col=milestone&order=priority +__ https://code.djangoproject.com/query?status=new&status=reopened&needs_better_patch=1&easy=1&col=id&col=summary&col=status&col=owner&col=type&col=milestone&order=priority +__ https://code.djangoproject.com/query?status=new&status=reopened&needs_tests=1&easy=1&col=id&col=summary&col=status&col=owner&col=type&col=milestone&order=priority + +What's next? +------------ + +After a ticket has a patch, it needs to be reviewed by a second set of eyes. +After uploading a patch or submitting a pull request, be sure to update the +ticket metadata by setting the flags on the ticket to say "has patch", +"doesn't need tests", etc, so others can find it for review. Contributing +doesn't necessarily always mean writing a patch from scratch. Reviewing +existing patches is also a very helpful contribution. See +:doc:`/internals/contributing/triaging-tickets` for details. diff --git a/docs/intro/index.txt b/docs/intro/index.txt index afb1825b87..bca2d7712b 100644 --- a/docs/intro/index.txt +++ b/docs/intro/index.txt @@ -15,6 +15,7 @@ place: read this material to quickly get up and running. tutorial04 reusable-apps whatsnext + contributing .. seealso:: From 1520748dac95a7f114e4bb2feeee04d46c720494 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Sat, 17 Nov 2012 20:24:54 +0100 Subject: [PATCH 151/302] =?UTF-8?q?Fixed=20#2550=20--=20Allow=20the=20auth?= =?UTF-8?q?=20backends=20to=20raise=20the=20PermissionDenied=20exception?= =?UTF-8?q?=20to=20completely=20stop=20the=20authentication=20chain.=20Man?= =?UTF-8?q?y=20thanks=20to=20namn,=20danielr,=20Dan=20Julius,=20=C5=81ukas?= =?UTF-8?q?z=20Rekucki,=20Aashu=20Dwivedi=20and=20umbrae=20for=20working?= =?UTF-8?q?=20this=20over=20the=20years.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- django/contrib/auth/__init__.py | 5 ++- django/contrib/auth/tests/auth_backends.py | 37 +++++++++++++++++++++- docs/releases/1.6.txt | 6 ++++ docs/topics/auth.txt | 6 ++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/django/contrib/auth/__init__.py b/django/contrib/auth/__init__.py index dd4a8484f5..5dbda44501 100644 --- a/django/contrib/auth/__init__.py +++ b/django/contrib/auth/__init__.py @@ -1,6 +1,6 @@ import re -from django.core.exceptions import ImproperlyConfigured +from django.core.exceptions import ImproperlyConfigured, PermissionDenied from django.utils.importlib import import_module from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed @@ -60,6 +60,9 @@ def authenticate(**credentials): except TypeError: # This backend doesn't accept these credentials as arguments. Try the next one. continue + except PermissionDenied: + # This backend says to stop in our tracks - this user should not be allowed in at all. + return None if user is None: continue # Annotate the user object with the path of the backend. diff --git a/django/contrib/auth/tests/auth_backends.py b/django/contrib/auth/tests/auth_backends.py index e92f159ff9..2ab0bd0efa 100644 --- a/django/contrib/auth/tests/auth_backends.py +++ b/django/contrib/auth/tests/auth_backends.py @@ -6,7 +6,8 @@ from django.contrib.auth.models import User, Group, Permission, AnonymousUser from django.contrib.auth.tests.utils import skipIfCustomUser from django.contrib.auth.tests.custom_user import ExtensionUser from django.contrib.contenttypes.models import ContentType -from django.core.exceptions import ImproperlyConfigured +from django.core.exceptions import ImproperlyConfigured, PermissionDenied +from django.contrib.auth import authenticate from django.test import TestCase from django.test.utils import override_settings @@ -323,3 +324,37 @@ class InActiveUserBackendTest(TestCase): def test_has_module_perms(self): self.assertEqual(self.user1.has_module_perms("app1"), False) self.assertEqual(self.user1.has_module_perms("app2"), False) + + +class PermissionDeniedBackend(object): + """ + Always raises PermissionDenied. + """ + supports_object_permissions = True + supports_anonymous_user = True + supports_inactive_user = True + + def authenticate(self, username=None, password=None): + raise PermissionDenied + + +class PermissionDeniedBackendTest(TestCase): + """ + Tests that other backends are not checked once a backend raises PermissionDenied + """ + backend = 'django.contrib.auth.tests.auth_backends.PermissionDeniedBackend' + + def setUp(self): + self.user1 = User.objects.create_user('test', 'test@example.com', 'test') + self.user1.save() + + @override_settings(AUTHENTICATION_BACKENDS=(backend, ) + + tuple(settings.AUTHENTICATION_BACKENDS)) + def test_permission_denied(self): + "user is not authenticated after a backend raises permission denied #2550" + self.assertEqual(authenticate(username='test', password='test'), None) + + @override_settings(AUTHENTICATION_BACKENDS=tuple( + settings.AUTHENTICATION_BACKENDS) + (backend, )) + def test_authenticates(self): + self.assertEqual(authenticate(username='test', password='test'), self.user1) diff --git a/docs/releases/1.6.txt b/docs/releases/1.6.txt index ef162a8de3..49649bc6b8 100644 --- a/docs/releases/1.6.txt +++ b/docs/releases/1.6.txt @@ -17,6 +17,12 @@ deprecation process for some features`_. What's new in Django 1.6 ======================== +Minor features +~~~~~~~~~~~~~~ + +* Authentication backends can raise ``PermissionDenied`` to immediately fail + the authentication chain. + Backwards incompatible changes in 1.6 ===================================== diff --git a/docs/topics/auth.txt b/docs/topics/auth.txt index aed482f710..6d8e3c66c3 100644 --- a/docs/topics/auth.txt +++ b/docs/topics/auth.txt @@ -2391,6 +2391,12 @@ processing at the first positive match. you need to force users to re-authenticate using different methods. A simple way to do that is simply to execute ``Session.objects.all().delete()``. +.. versionadded:: 1.6 + +If a backend raises a :class:`~django.core.exceptions.PermissionDenied` +exception, authentication will immediately fail. Django won't check the +backends that follow. + Writing an authentication backend --------------------------------- From 9b755a298a55849bf8e831a46639999609aedfff Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 17 Nov 2012 22:38:19 +0100 Subject: [PATCH 152/302] Fixed #19291 -- Completed deprecation of ADMIN_MEDIA_PREFIX. --- django/conf/__init__.py | 3 --- django/contrib/admin/templatetags/adminmedia.py | 15 --------------- docs/internals/deprecation.txt | 9 +++++---- docs/ref/settings.txt | 10 ---------- docs/releases/1.5.txt | 5 +++++ 5 files changed, 10 insertions(+), 32 deletions(-) delete mode 100644 django/contrib/admin/templatetags/adminmedia.py diff --git a/django/conf/__init__.py b/django/conf/__init__.py index dec4cf9418..b00c8d5046 100644 --- a/django/conf/__init__.py +++ b/django/conf/__init__.py @@ -110,9 +110,6 @@ class BaseSettings(object): def __setattr__(self, name, value): if name in ("MEDIA_URL", "STATIC_URL") and value and not value.endswith('/'): raise ImproperlyConfigured("If set, %s must end with a slash" % name) - elif name == "ADMIN_MEDIA_PREFIX": - warnings.warn("The ADMIN_MEDIA_PREFIX setting has been removed; " - "use STATIC_URL instead.", DeprecationWarning) elif name == "ALLOWED_INCLUDE_ROOTS" and isinstance(value, six.string_types): raise ValueError("The ALLOWED_INCLUDE_ROOTS setting must be set " "to a tuple, not a string.") diff --git a/django/contrib/admin/templatetags/adminmedia.py b/django/contrib/admin/templatetags/adminmedia.py deleted file mode 100644 index b08d13c18f..0000000000 --- a/django/contrib/admin/templatetags/adminmedia.py +++ /dev/null @@ -1,15 +0,0 @@ -import warnings -from django.template import Library -from django.templatetags.static import PrefixNode - -register = Library() - -@register.simple_tag -def admin_media_prefix(): - """ - Returns the string contained in the setting ADMIN_MEDIA_PREFIX. - """ - warnings.warn( - "The admin_media_prefix template tag is deprecated. " - "Use the static template tag instead.", DeprecationWarning) - return PrefixNode.handle_simple("ADMIN_MEDIA_PREFIX") diff --git a/docs/internals/deprecation.txt b/docs/internals/deprecation.txt index 9fd92db2b4..414da30ff8 100644 --- a/docs/internals/deprecation.txt +++ b/docs/internals/deprecation.txt @@ -140,6 +140,11 @@ these changes. removed. In its place use :class:`~django.contrib.staticfiles.handlers.StaticFilesHandler`. +* The template tags library ``adminmedia`` and the template tag ``{% + admin_media_prefix %}`` will be removed in favor of the generic static files + handling. (This is faster than the usual deprecation path; see the + :doc:`Django 1.4 release notes`.) + * The :ttag:`url` and :ttag:`ssi` template tags will be modified so that the first argument to each tag is a template variable, not an implied string. In 1.4, this behavior is provided by a version of the tag @@ -232,10 +237,6 @@ these changes. :setting:`LOGGING` setting should include this filter explicitly if it is desired. -* The template tag - :func:`django.contrib.admin.templatetags.adminmedia.admin_media_prefix` - will be removed in favor of the generic static files handling. - * The builtin truncation functions :func:`django.utils.text.truncate_words` and :func:`django.utils.text.truncate_html_words` will be removed in favor of the ``django.utils.text.Truncator`` class. diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 5544c99dd1..9b222bf586 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -2210,16 +2210,6 @@ The default value for the X-Frame-Options header used by Deprecated settings =================== -.. setting:: ADMIN_MEDIA_PREFIX - -ADMIN_MEDIA_PREFIX ------------------- - -.. deprecated:: 1.4 - This setting has been obsoleted by the ``django.contrib.staticfiles`` app - integration. See the :doc:`Django 1.4 release notes` for - more information. - .. setting:: AUTH_PROFILE_MODULE AUTH_PROFILE_MODULE diff --git a/docs/releases/1.5.txt b/docs/releases/1.5.txt index c53518feaa..cffc0f23af 100644 --- a/docs/releases/1.5.txt +++ b/docs/releases/1.5.txt @@ -574,6 +574,11 @@ Miscellaneous HTML validation against pre-HTML5 Strict DTDs, you should add a div around it in your pages. +* The template tags library ``adminmedia``, which only contained the + deprecated template tag ``{% admin_media_prefix %}``, was removed. + Attempting to load it with ``{% load adminmedia %}`` will fail. If your + templates still contain that line you must remove it. + Features deprecated in 1.5 ========================== From 4585e123187a8a94a0db11d11358398911b10168 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 17 Nov 2012 23:25:37 +0100 Subject: [PATCH 153/302] Fix typo in file storage docs. --- docs/howto/custom-file-storage.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/howto/custom-file-storage.txt b/docs/howto/custom-file-storage.txt index 51558d9715..090cc089cb 100644 --- a/docs/howto/custom-file-storage.txt +++ b/docs/howto/custom-file-storage.txt @@ -27,7 +27,7 @@ You'll need to follow these steps: option = settings.CUSTOM_STORAGE_OPTIONS ... -#. Your storage class must implement the :meth:`_open()` and :meth:`_save() +#. Your storage class must implement the :meth:`_open()` and :meth:`_save()` methods, along with any other methods appropriate to your storage class. See below for more on these methods. From ccb2b574e8ea961f11b1c36bcbdf396cd9acb550 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 17 Nov 2012 23:25:52 +0100 Subject: [PATCH 154/302] Fixed #19315 -- Improved markup in admin FAQ. Thanks ClaesBas. --- docs/faq/admin.txt | 23 +++++++++++++---------- docs/ref/contrib/admin/index.txt | 2 ++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/faq/admin.txt b/docs/faq/admin.txt index 872ad254c9..30d452cbe2 100644 --- a/docs/faq/admin.txt +++ b/docs/faq/admin.txt @@ -10,8 +10,8 @@ things: * Set the :setting:`SESSION_COOKIE_DOMAIN` setting in your admin config file to match your domain. For example, if you're going to - "http://www.example.com/admin/" in your browser, in - "myproject.settings" you should set ``SESSION_COOKIE_DOMAIN = 'www.example.com'``. + "http://www.example.com/admin/" in your browser, in "myproject.settings" you + should set :setting:`SESSION_COOKIE_DOMAIN` = 'www.example.com'. * Some browsers (Firefox?) don't like to accept cookies from domains that don't have dots in them. If you're running the admin site on "localhost" @@ -23,8 +23,9 @@ I can't log in. When I enter a valid username and password, it brings up the log ----------------------------------------------------------------------------------------------------------------------------------------------------------- If you're sure your username and password are correct, make sure your user -account has ``is_active`` and ``is_staff`` set to True. The admin site only -allows access to users with those two fields both set to True. +account has :attr:`~django.contrib.auth.models.User.is_active` and +:attr:`~django.contrib.auth.models.User.is_staff` set to True. The admin site +only allows access to users with those two fields both set to True. How can I prevent the cache middleware from caching the admin site? ------------------------------------------------------------------- @@ -64,9 +65,10 @@ My "list_filter" contains a ManyToManyField, but the filter doesn't display. Django won't bother displaying the filter for a ``ManyToManyField`` if there are fewer than two related objects. -For example, if your ``list_filter`` includes ``sites``, and there's only one -site in your database, it won't display a "Site" filter. In that case, -filtering by site would be meaningless. +For example, if your :attr:`~django.contrib.admin.ModelAdmin.list_filter` +includes :doc:`sites `, and there's only one site in your +database, it won't display a "Site" filter. In that case, filtering by site +would be meaningless. Some objects aren't appearing in the admin. ------------------------------------------- @@ -85,9 +87,10 @@ How can I customize the functionality of the admin interface? You've got several options. If you want to piggyback on top of an add/change form that Django automatically generates, you can attach arbitrary JavaScript -modules to the page via the model's ``class Admin`` ``js`` parameter. That -parameter is a list of URLs, as strings, pointing to JavaScript modules that -will be included within the admin form via a ``