Cleaned up the template debug implementation.

This patch does three major things:

* Merges the django.template.debug implementation into django.template.base.

* Simplifies the debug implementation.

  The old implementation copied debug information to every token and node.
  The django_template_source attribute was set in multiple places, some
  quite hacky, like django.template.defaulttags.ForNode.

  Debug information is now annotated in two high-level places:

  * Template.compile_nodelist for errors during parsing
  * Node.render_annotated for errors during rendering

  These were chosen because they have access to the template and context
  as well as to all exceptions that happen during either the parse or
  render phase.

* Moves the contextual line traceback information creation from
  django.views.debug into django.template.base.Template. The debug views now
  only deal with the presentation of the debug information.

tests/template_tests/syntax_tests/test_basic.py

@@ -311,3 +311,15 @@ class BasicSyntaxTests(SimpleTestCase):
         """
         output = self.engine.render_to_string('basic-syntax38', {"var": {"callable": lambda: "foo bar"}})
         self.assertEqual(output, 'foo bar')
+
+    @setup({'template': '{% block content %}'})
+    def test_unclosed_block(self):
+        msg = "Unclosed tag 'block'. Looking for one of: endblock."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')
+
+    @setup({'template': '{% if a %}'})
+    def test_unclosed_block2(self):
+        msg = "Unclosed tag 'if'. Looking for one of: elif, else, endif."
+        with self.assertRaisesMessage(TemplateSyntaxError, msg):
+            self.engine.render_to_string('template')

tests/template_tests/test_engine.py

@@ -52,23 +52,11 @@ class DeprecatedRenderToStringTest(SimpleTestCase):
 
 class LoaderTests(SimpleTestCase):
 
-    def test_debug_nodelist_name(self):
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
-        template_name = 'index.html'
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(name.endswith(template_name))
-
     def test_origin(self):
         engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
         template = engine.get_template('index.html')
         self.assertEqual(template.origin.loadname, 'index.html')
 
-    def test_origin_debug_false(self):
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=False)
-        template = engine.get_template('index.html')
-        self.assertEqual(template.origin, None)
-
     def test_loader_priority(self):
         """
         #21460 -- Check that the order of template loader works.

tests/template_tests/test_loaders.py

@@ -64,24 +64,6 @@ class CachedLoaderTests(SimpleTestCase):
             "Cached loader failed to cache the TemplateDoesNotExist exception",
         )
 
-    def test_debug_nodelist_name(self):
-        template_name = 'index.html'
-        engine = Engine(dirs=[TEMPLATE_DIR], debug=True)
-
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(
-            name.endswith(template_name),
-            'Template loaded through cached loader has incorrect name for debug page: %s' % template_name,
-        )
-
-        template = engine.get_template(template_name)
-        name = template.nodelist[0].source[0].name
-        self.assertTrue(
-            name.endswith(template_name),
-            'Cached template loaded through cached loader has incorrect name for debug page: %s' % template_name,
-        )
-
 
 @unittest.skipUnless(pkg_resources, 'setuptools is not installed')
 class EggLoaderTests(SimpleTestCase):

tests/template_tests/test_nodelist.py

@@ -51,6 +51,5 @@ class ErrorIndexTest(TestCase):
             try:
                 template.render(context)
             except (RuntimeError, TypeError) as e:
-                error_source_index = e.django_template_source[1]
-                self.assertEqual(error_source_index,
-                                 expected_error_source_index)
+                debug = e.template_debug
+                self.assertEqual((debug['start'], debug['end']), expected_error_source_index)

tests/template_tests/test_parser.py

@@ -5,11 +5,10 @@ from __future__ import unicode_literals
 
 from unittest import TestCase
 
-from django.template import Library, Template, TemplateSyntaxError
+from django.template import Library, TemplateSyntaxError
 from django.template.base import (
     TOKEN_BLOCK, FilterExpression, Parser, Token, Variable,
 )
-from django.test import override_settings
 from django.utils import six
 
 
@@ -73,14 +72,6 @@ class ParserTests(TestCase):
         with six.assertRaisesRegex(self, TypeError, "Variable must be a string or number, got <(class|type) 'dict'>"):
             Variable({})
 
-    @override_settings(DEBUG=True)
-    def test_compile_filter_error(self):
-        # regression test for #19819
-        msg = "Could not parse the remainder: '@bar' from 'foo@bar'"
-        with six.assertRaisesRegex(self, TemplateSyntaxError, msg) as cm:
-            Template("{% if 1 %}{{ foo@bar }}{% endif %}")
-        self.assertEqual(cm.exception.django_template_source[1], (10, 23))
-
     def test_filter_args_count(self):
         p = Parser("")
         l = Library()

tests/template_tests/tests.py

@@ -6,7 +6,7 @@ import sys
 from django.contrib.auth.models import Group
 from django.core import urlresolvers
 from django.template import (
-    Context, Template, TemplateSyntaxError, engines, loader,
+    Context, Engine, Template, TemplateSyntaxError, engines, loader,
 )
 from django.test import SimpleTestCase, override_settings
 
@@ -48,31 +48,61 @@ class TemplateTests(SimpleTestCase):
             self.assertGreater(depth, 5,
                 "The traceback context was lost when reraising the traceback. See #19827")
 
-    @override_settings(DEBUG=True)
     def test_no_wrapped_exception(self):
         """
         # 16770 -- The template system doesn't wrap exceptions, but annotates
         them.
         """
+        engine = Engine(debug=True)
         c = Context({"coconuts": lambda: 42 / 0})
-        t = Template("{{ coconuts }}")
-        with self.assertRaises(ZeroDivisionError) as cm:
+        t = engine.from_string("{{ coconuts }}")
+
+        with self.assertRaises(ZeroDivisionError) as e:
             t.render(c)
 
-        self.assertEqual(cm.exception.django_template_source[1], (0, 14))
+        debug = e.exception.template_debug
+        self.assertEqual(debug['start'], 0)
+        self.assertEqual(debug['end'], 14)
 
     def test_invalid_block_suggestion(self):
         """
         #7876 -- Error messages should include the unexpected block name.
         """
+        engine = Engine()
+
         with self.assertRaises(TemplateSyntaxError) as e:
-            Template("{% if 1 %}lala{% endblock %}{% endif %}")
+            engine.from_string("{% if 1 %}lala{% endblock %}{% endif %}")
 
         self.assertEqual(
             e.exception.args[0],
             "Invalid block tag: 'endblock', expected 'elif', 'else' or 'endif'",
         )
 
+    def test_compile_filter_expression_error(self):
+        """
+        19819 -- Make sure the correct token is highlighted for
+        FilterExpression errors.
+        """
+        engine = Engine(debug=True)
+        msg = "Could not parse the remainder: '@bar' from 'foo@bar'"
+
+        with self.assertRaisesMessage(TemplateSyntaxError, msg) as e:
+            engine.from_string("{% if 1 %}{{ foo@bar }}{% endif %}")
+
+        debug = e.exception.template_debug
+        self.assertEqual((debug['start'], debug['end']), (10, 23))
+        self.assertEqual((debug['during']), '{{ foo@bar }}')
+
+    def test_compile_tag_error(self):
+        """
+        Errors raised while compiling nodes should include the token
+        information.
+        """
+        engine = Engine(debug=True)
+        with self.assertRaises(RuntimeError) as e:
+            engine.from_string("{% load bad_tag %}{% badtag %}")
+        self.assertEqual(e.exception.template_debug['during'], '{% badtag %}')
+
     def test_super_errors(self):
         """
         #18169 -- NoReverseMatch should not be silence in block.super.