django/tests/admin_scripts/tests.py

# -*- coding: utf-8 -*-
"""
A series of tests to establish that the command-line management tools work as
advertised - especially with regards to the handling of the
DJANGO_SETTINGS_MODULE and default settings.py files.
"""
from __future__ import unicode_literals

import codecs
import os
import re
import shutil
import socket
import subprocess
import sys
import tempfile
import unittest

import django
from django import conf, get_version
from django.conf import settings
from django.core.management import (
    BaseCommand, CommandError, call_command, color,
)
from django.db import ConnectionHandler
from django.db.migrations.exceptions import MigrationSchemaMissing
from django.db.migrations.recorder import MigrationRecorder
from django.test import (
    LiveServerTestCase, SimpleTestCase, TestCase, mock, override_settings,
)
from django.test.runner import DiscoverRunner
from django.utils._os import npath, upath
from django.utils.encoding import force_text
from django.utils.six import PY2, PY3, StringIO

custom_templates_dir = os.path.join(os.path.dirname(upath(__file__)), 'custom_templates')

SYSTEM_CHECK_MSG = 'System check identified no issues'


class AdminScriptTestCase(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        super(AdminScriptTestCase, cls).setUpClass()
        cls.test_dir = os.path.realpath(os.path.join(
            tempfile.gettempdir(),
            cls.__name__,
            'test_project',
        ))
        if not os.path.exists(cls.test_dir):
            os.makedirs(cls.test_dir)
        with open(os.path.join(cls.test_dir, '__init__.py'), 'w'):
            pass

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.test_dir)
        super(AdminScriptTestCase, cls).tearDownClass()

    def write_settings(self, filename, apps=None, is_dir=False, sdict=None, extra=None):
        if is_dir:
            settings_dir = os.path.join(self.test_dir, filename)
            os.mkdir(settings_dir)
            settings_file_path = os.path.join(settings_dir, '__init__.py')
        else:
            settings_file_path = os.path.join(self.test_dir, filename)

        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# -*- coding: utf-8 -*\n')
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            if extra:
                settings_file.write("%s\n" % extra)
            exports = [
                'DATABASES',
                'ROOT_URLCONF',
                'SECRET_KEY',
            ]
            for s in exports:
                if hasattr(settings, s):
                    o = getattr(settings, s)
                    if not isinstance(o, (dict, tuple, list)):
                        o = "'%s'" % o
                    settings_file.write("%s = %s\n" % (s, o))

            if apps is None:
                apps = ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts']

            settings_file.write("INSTALLED_APPS = %s\n" % apps)

            if sdict:
                for k, v in sdict.items():
                    settings_file.write("%s = %s\n" % (k, v))

    def remove_settings(self, filename, is_dir=False):
        full_name = os.path.join(self.test_dir, filename)
        if is_dir:
            shutil.rmtree(full_name)
        else:
            os.remove(full_name)

        # Also try to remove the compiled file; if it exists, it could
        # mess up later tests that depend upon the .py file not existing
        try:
            if sys.platform.startswith('java'):
                # Jython produces module$py.class files
                os.remove(re.sub(r'\.py$', '$py.class', full_name))
            else:
                # CPython produces module.pyc files
                os.remove(full_name + 'c')
        except OSError:
            pass
        # Also remove a __pycache__ directory, if it exists
        cache_name = os.path.join(self.test_dir, '__pycache__')
        if os.path.isdir(cache_name):
            shutil.rmtree(cache_name)

    def _ext_backend_paths(self):
        """
        Returns the paths for any external backend packages.
        """
        paths = []
        first_package_re = re.compile(r'(^[^\.]+)\.')
        for backend in settings.DATABASES.values():
            result = first_package_re.findall(backend['ENGINE'])
            if result and result != ['django']:
                backend_pkg = __import__(result[0])
                backend_dir = os.path.dirname(backend_pkg.__file__)
                paths.append(os.path.dirname(backend_dir))
        return paths

    def run_test(self, script, args, settings_file=None, apps=None):
        base_dir = os.path.dirname(self.test_dir)
        # The base dir for Django's tests is one level up.
        tests_dir = os.path.dirname(os.path.dirname(upath(__file__)))
        # The base dir for Django is one level above the test dir. We don't use
        # `import django` to figure that out, so we don't pick up a Django
        # from site-packages or similar.
        django_dir = os.path.dirname(tests_dir)
        ext_backend_base_dirs = self._ext_backend_paths()

        # Define a temporary environment for the subprocess
        test_environ = os.environ.copy()
        if sys.platform.startswith('java'):
            python_path_var_name = 'JYTHONPATH'
        else:
            python_path_var_name = 'PYTHONPATH'

        old_cwd = os.getcwd()

        # Set the test environment
        if settings_file:
            test_environ['DJANGO_SETTINGS_MODULE'] = str(settings_file)
        elif 'DJANGO_SETTINGS_MODULE' in test_environ:
            del test_environ['DJANGO_SETTINGS_MODULE']
        python_path = [base_dir, django_dir, tests_dir]
        python_path.extend(ext_backend_base_dirs)
        # Use native strings for better compatibility
        test_environ[str(python_path_var_name)] = npath(os.pathsep.join(python_path))
        test_environ[str('PYTHONWARNINGS')] = str('')

        # Move to the test directory and run
        os.chdir(self.test_dir)
        out, err = subprocess.Popen([sys.executable, script] + args,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                env=test_environ, universal_newlines=True).communicate()
        # Move back to the old working directory
        os.chdir(old_cwd)

        return out, err

    def run_django_admin(self, args, settings_file=None):
        script_dir = os.path.abspath(os.path.join(os.path.dirname(upath(django.__file__)), 'bin'))
        return self.run_test(os.path.join(script_dir, 'django-admin.py'), args, settings_file)

    def run_manage(self, args, settings_file=None):
        def safe_remove(path):
            try:
                os.remove(path)
            except OSError:
                pass

        conf_dir = os.path.dirname(upath(conf.__file__))
        template_manage_py = os.path.join(conf_dir, 'project_template', 'manage.py-tpl')

        test_manage_py = os.path.join(self.test_dir, 'manage.py')
        shutil.copyfile(template_manage_py, test_manage_py)

        with open(test_manage_py, 'r') as fp:
            manage_py_contents = fp.read()
        manage_py_contents = manage_py_contents.replace(
            "{{ project_name }}", "test_project")
        with open(test_manage_py, 'w') as fp:
            fp.write(manage_py_contents)
        self.addCleanup(safe_remove, test_manage_py)

        return self.run_test('./manage.py', args, settings_file)

    def assertNoOutput(self, stream):
        "Utility assertion: assert that the given stream is empty"
        self.assertEqual(len(stream), 0, "Stream should be empty: actually contains '%s'" % stream)

    def assertOutput(self, stream, msg, regex=False):
        "Utility assertion: assert that the given message exists in the output"
        stream = force_text(stream)
        if regex:
            self.assertIsNotNone(re.search(msg, stream),
                "'%s' does not match actual output text '%s'" % (msg, stream))
        else:
            self.assertIn(msg, stream, "'%s' does not match actual output text '%s'" % (msg, stream))

    def assertNotInOutput(self, stream, msg):
        "Utility assertion: assert that the given message doesn't exist in the output"
        stream = force_text(stream)
        self.assertNotIn(msg, stream, "'%s' matches actual output text '%s'" % (msg, stream))

##########################################################################
# DJANGO ADMIN TESTS
# This first series of test classes checks the environment processing
# of the django-admin.py script
##########################################################################


class DjangoAdminNoSettings(AdminScriptTestCase):
    "A series of tests for django-admin.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "no settings: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)


class DjangoAdminDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "default: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: django-admin can't execute user commands if it isn't provided settings"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "default: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes',
                                            'admin_scripts', 'admin_scripts.complex_app'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "fulldefault: django-admin builtin commands succeed if a settings file is provided"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: django-admin builtin commands succeed if the environment contains settings"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "fulldefault: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMinimalSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "minimal: django-admin builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: django-admin builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: django-admin can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: django-admin can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class DjangoAdminAlternateSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminMultipleSettings(AdminScriptTestCase):
    """A series of tests for django-admin.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_settings(self):
        "alternate: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.alternate_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "alternate: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "alternate: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "alternate: django-admin can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.alternate_settings']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "alternate: django-admin can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_django_admin(args, 'test_project.alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class DjangoAdminSettingsDirectory(AdminScriptTestCase):
    """
    A series of tests for django-admin.py when the settings file is in a
    directory. (see #9751).
    """

    def setUp(self):
        self.write_settings('settings', is_dir=True)

    def tearDown(self):
        self.remove_settings('settings', is_dir=True)

    def test_setup_environ(self):
        "directory: startapp creates the correct directory"
        args = ['startapp', 'settings_test']
        app_path = os.path.join(self.test_dir, 'settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        unicode_literals_import = "from __future__ import unicode_literals\n"
        with open(os.path.join(app_path, 'apps.py'), 'r') as f:
            content = f.read()
            self.assertIn("class SettingsTestConfig(AppConfig)", content)
            self.assertIn("name = 'settings_test'", content)
            if not PY3:
                self.assertIn(unicode_literals_import, content)
        if not PY3:
            with open(os.path.join(app_path, 'models.py'), 'r') as fp:
                content = fp.read()
            self.assertIn(unicode_literals_import, content)

    def test_setup_environ_custom_template(self):
        "directory: startapp creates the correct directory with a custom template"
        template_path = os.path.join(custom_templates_dir, 'app_template')
        args = ['startapp', '--template', template_path, 'custom_settings_test']
        app_path = os.path.join(self.test_dir, 'custom_settings_test')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        self.assertTrue(os.path.exists(os.path.join(app_path, 'api.py')))

    @unittest.skipIf(PY2, "Python 2 doesn't support Unicode package names.")
    def test_startapp_unicode_name(self):
        "directory: startapp creates the correct directory with unicode characters"
        args = ['startapp', 'こんにちは']
        app_path = os.path.join(self.test_dir, 'こんにちは')
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.addCleanup(shutil.rmtree, app_path)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(app_path))
        with open(os.path.join(app_path, 'apps.py'), 'r', encoding='utf8') as f:
            content = f.read()
            self.assertIn("class こんにちはConfig(AppConfig)", content)
            self.assertIn("name = 'こんにちは'", content)

    def test_builtin_command(self):
        "directory: django-admin builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'settings are not configured')

    def test_builtin_with_bad_settings(self):
        "directory: django-admin builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "directory: django-admin builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "directory: django-admin can't execute user commands unless settings are provided"
        args = ['noargs_command']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No Django settings specified")
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_builtin_with_settings(self):
        "directory: django-admin builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "directory: django-admin builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_django_admin(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)


##########################################################################
# MANAGE.PY TESTS
# This next series of test classes checks the environment processing
# of the generated manage.py script
##########################################################################

class ManageNoSettings(AdminScriptTestCase):
    "A series of tests for manage.py when there is no settings.py file."

    def test_builtin_command(self):
        "no settings: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_bad_settings(self):
        "no settings: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "no settings: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)


class ManageDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application.
    """
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "default: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "default: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "default: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "default: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "default: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "default: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "default: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "default: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageFullPathDefaultSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    contains the test application specified using a full path.
    """
    def setUp(self):
        self.write_settings('settings.py', ['django.contrib.auth', 'django.contrib.contenttypes', 'admin_scripts'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "fulldefault: manage.py builtin commands succeed when default settings are appropriate"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "fulldefault: manage.py builtin commands succeed if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "fulldefault: manage.py builtin commands succeed if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "fulldefault: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "fulldefault: manage.py can execute user commands when default settings are appropriate"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_settings(self):
        "fulldefault: manage.py can execute user commands when settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "fulldefault: manage.py can execute user commands when settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageMinimalSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings.py file that
    doesn't contain the test application.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_builtin_command(self):
        "minimal: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "minimal: manage.py builtin commands fail if settings are provided as argument"
        args = ['check', '--settings=test_project.settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_environment(self):
        "minimal: manage.py builtin commands fail if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_bad_settings(self):
        "minimal: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "minimal: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "minimal: manage.py can't execute user commands without appropriate settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "minimal: manage.py can't execute user commands, even if settings are provided as argument"
        args = ['noargs_command', '--settings=test_project.settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_environment(self):
        "minimal: manage.py can't execute user commands, even if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'test_project.settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")


class ManageAlternateSettings(AdminScriptTestCase):
    """A series of tests for manage.py when using a settings file
    with a name other than 'settings.py'.
    """
    def setUp(self):
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "alternate: manage.py builtin commands fail with an error when no default settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_builtin_with_settings(self):
        "alternate: manage.py builtin commands work with settings provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_environment(self):
        "alternate: manage.py builtin commands work if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(out, SYSTEM_CHECK_MSG)
        self.assertNoOutput(err)

    def test_builtin_with_bad_settings(self):
        "alternate: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "alternate: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "alternate: manage.py can't execute user commands without settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?(test_project\.)?settings'?", regex=True)

    def test_custom_command_with_settings(self):
        "alternate: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_with_environment(self):
        "alternate: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )
        self.assertNoOutput(err)

    def test_custom_command_output_color(self):
        "alternate: manage.py output syntax color can be deactivated with the `--no-color` option"
        args = ['noargs_command', '--no-color', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', True), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]"
        )
        self.assertNoOutput(err)


class ManageMultipleSettings(AdminScriptTestCase):
    """A series of tests for manage.py when multiple settings files
    (including the default 'settings.py') are available. The default settings
    file is insufficient for performing the operations described, so the
    alternate settings must be used by the running script.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_builtin_command(self):
        "multiple: manage.py builtin commands fail with an error when no settings provided"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No installed app with label 'admin_scripts'.")

    def test_builtin_with_settings(self):
        "multiple: manage.py builtin commands succeed if settings are provided as argument"
        args = ['check', '--settings=alternate_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_environment(self):
        "multiple: manage.py can execute builtin commands if settings are provided in the environment"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, SYSTEM_CHECK_MSG)

    def test_builtin_with_bad_settings(self):
        "multiple: manage.py builtin commands fail if settings file (from argument) doesn't exist"
        args = ['check', '--settings=bad_settings', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_builtin_with_bad_environment(self):
        "multiple: manage.py builtin commands fail if settings file (from environment) doesn't exist"
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args, 'bad_settings')
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named '?bad_settings'?", regex=True)

    def test_custom_command(self):
        "multiple: manage.py can't execute user commands using default settings"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Unknown command: 'noargs_command'")

    def test_custom_command_with_settings(self):
        "multiple: manage.py can execute user commands if settings are provided as argument"
        args = ['noargs_command', '--settings=alternate_settings']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")

    def test_custom_command_with_environment(self):
        "multiple: manage.py can execute user commands if settings are provided in environment"
        args = ['noargs_command']
        out, err = self.run_manage(args, 'alternate_settings')
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE: noargs_command")


class ManageSettingsWithSettingsErrors(AdminScriptTestCase):
    """
    Tests for manage.py when using the default settings.py file containing
    runtime errors.
    """
    def tearDown(self):
        self.remove_settings('settings.py')

    def write_settings_with_import_error(self, filename):
        settings_file_path = os.path.join(self.test_dir, filename)
        with open(settings_file_path, 'w') as settings_file:
            settings_file.write('# Settings file automatically generated by admin_scripts test case\n')
            settings_file.write('# The next line will cause an import error:\nimport foo42bar\n')

    def test_import_error(self):
        """
        import error: manage.py builtin commands shows useful diagnostic info
        when settings with import errors is provided (#14130).
        """
        self.write_settings_with_import_error('settings.py')
        args = ['check', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "No module named")
        self.assertOutput(err, "foo42bar")

    def test_attribute_error(self):
        """
        manage.py builtin commands does not swallow attribute error due to bad
        settings (#18845).
        """
        self.write_settings('settings.py', sdict={'BAD_VAR': 'INSTALLED_APPS.crash'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "AttributeError: 'list' object has no attribute 'crash'")

    def test_key_error(self):
        self.write_settings('settings.py', sdict={'BAD_VAR': 'DATABASES["blah"]'})
        args = ['collectstatic', 'admin_scripts']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "KeyError: 'blah'")

    def test_help(self):
        """
        Test listing available commands output note when only core commands are
        available.
        """
        self.write_settings('settings.py', sdict={'MEDIA_URL': '"/no_ending_slash"'})
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, 'only Django core commands are listed')
        self.assertNoOutput(err)


class ManageCheck(AdminScriptTestCase):
    def tearDown(self):
        self.remove_settings('settings.py')

    def test_nonexistent_app(self):
        """ manage.py check reports an error on a non-existent app in
        INSTALLED_APPS """

        self.write_settings('settings.py',
            apps=['admin_scriptz.broken_app'],
            sdict={'USE_I18N': False})
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')
        self.assertOutput(err, 'No module named')
        self.assertOutput(err, 'admin_scriptz')

    def test_broken_app(self):
        """ manage.py check reports an ImportError if an app's models.py
        raises one on import """

        self.write_settings('settings.py', apps=['admin_scripts.broken_app'])
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, 'ImportError')

    def test_complex_app(self):
        """ manage.py check does not raise an ImportError validating a
        complex app with nested calls to load_app """

        self.write_settings(
            'settings.py',
            apps=[
                'admin_scripts.complex_app',
                'admin_scripts.simple_app',
                'django.contrib.admin.apps.SimpleAdminConfig',
                'django.contrib.auth',
                'django.contrib.contenttypes',
            ],
            sdict={
                'DEBUG': True
            }
        )
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_app_with_import(self):
        """ manage.py check does not raise errors when an app imports a base
        class that itself has an abstract base. """

        self.write_settings('settings.py',
            apps=['admin_scripts.app_with_import',
                  'django.contrib.auth',
                  'django.contrib.contenttypes',
                  'django.contrib.sites'],
            sdict={'DEBUG': True})
        args = ['check']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertEqual(out, 'System check identified no issues (0 silenced).\n')

    def test_output_format(self):
        """ All errors/warnings should be sorted by level and by message. """

        self.write_settings('settings.py',
            apps=['admin_scripts.app_raising_messages',
                  'django.contrib.auth',
                  'django.contrib.contenttypes'],
            sdict={'DEBUG': True})
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "SystemCheckError: System check identified some issues:\n"
            "\n"
            "ERRORS:\n"
            "?: An error\n"
            "\tHINT: Error hint\n"
            "\n"
            "WARNINGS:\n"
            "a: Second warning\n"
            "obj: First warning\n"
            "\tHINT: Hint\n"
            "\n"
            "System check identified 3 issues (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)

    def test_warning_does_not_halt(self):
        """
        When there are only warnings or less serious messages, then Django
        should not prevent user from launching their project, so `check`
        command should not raise `CommandError` exception.

        In this test we also test output format.
        """

        self.write_settings('settings.py',
            apps=['admin_scripts.app_raising_warning',
                  'django.contrib.auth',
                  'django.contrib.contenttypes'],
            sdict={'DEBUG': True})
        args = ['check']
        out, err = self.run_manage(args)
        expected_err = (
            "System check identified some issues:\n"  # No "CommandError: " part
            "\n"
            "WARNINGS:\n"
            "?: A warning\n"
            "\n"
            "System check identified 1 issue (0 silenced).\n"
        )
        self.assertEqual(err, expected_err)
        self.assertNoOutput(out)


class CustomTestRunner(DiscoverRunner):

    def __init__(self, *args, **kwargs):
        assert 'liveserver' not in kwargs
        super(CustomTestRunner, self).__init__(*args, **kwargs)

    def run_tests(self, test_labels, extra_tests=None, **kwargs):
        pass


class ManageTestCommand(AdminScriptTestCase):
    def test_liveserver(self):
        """
        Ensure that the --liveserver option sets the environment variable
        correctly.
        Refs #2879.
        """

        # Backup original state
        address_predefined = 'DJANGO_LIVE_TEST_SERVER_ADDRESS' in os.environ
        old_address = os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS')

        call_command('test', verbosity=0, testrunner='admin_scripts.tests.CustomTestRunner')

        # Original state hasn't changed
        self.assertEqual('DJANGO_LIVE_TEST_SERVER_ADDRESS' in os.environ, address_predefined)
        self.assertEqual(os.environ.get('DJANGO_LIVE_TEST_SERVER_ADDRESS'), old_address)

        call_command('test', verbosity=0, testrunner='admin_scripts.tests.CustomTestRunner', liveserver='blah')

        # Variable was correctly set
        self.assertEqual(os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'], 'blah')

        # Restore original state
        if address_predefined:
            os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS'] = old_address
        else:
            del os.environ['DJANGO_LIVE_TEST_SERVER_ADDRESS']


class ManageRunserver(AdminScriptTestCase):
    def setUp(self):
        from django.core.management.commands.runserver import Command

        def monkey_run(*args, **options):
            return

        self.output = StringIO()
        self.cmd = Command(stdout=self.output)
        self.cmd.run = monkey_run

    def assertServerSettings(self, addr, port, ipv6=False, raw_ipv6=False):
        self.assertEqual(self.cmd.addr, addr)
        self.assertEqual(self.cmd.port, port)
        self.assertEqual(self.cmd.use_ipv6, ipv6)
        self.assertEqual(self.cmd._raw_ipv6, raw_ipv6)

    def test_runserver_addrport(self):
        call_command(self.cmd)
        self.assertServerSettings('127.0.0.1', '8000')

        call_command(self.cmd, addrport="1.2.3.4:8000")
        self.assertServerSettings('1.2.3.4', '8000')

        call_command(self.cmd, addrport="7000")
        self.assertServerSettings('127.0.0.1', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_addrport_ipv6(self):
        call_command(self.cmd, addrport="", use_ipv6=True)
        self.assertServerSettings('::1', '8000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="7000", use_ipv6=True)
        self.assertServerSettings('::1', '7000', ipv6=True, raw_ipv6=True)

        call_command(self.cmd, addrport="[2001:0db8:1234:5678::9]:7000")
        self.assertServerSettings('2001:0db8:1234:5678::9', '7000', ipv6=True, raw_ipv6=True)

    def test_runner_hostname(self):
        call_command(self.cmd, addrport="localhost:8000")
        self.assertServerSettings('localhost', '8000')

        call_command(self.cmd, addrport="test.domain.local:7000")
        self.assertServerSettings('test.domain.local', '7000')

    @unittest.skipUnless(socket.has_ipv6, "platform doesn't support IPv6")
    def test_runner_hostname_ipv6(self):
        call_command(self.cmd, addrport="test.domain.local:7000", use_ipv6=True)
        self.assertServerSettings('test.domain.local', '7000', ipv6=True)

    def test_runner_ambiguous(self):
        # Only 4 characters, all of which could be in an ipv6 address
        call_command(self.cmd, addrport="beef:7654")
        self.assertServerSettings('beef', '7654')

        # Uses only characters that could be in an ipv6 address
        call_command(self.cmd, addrport="deadbeef:7654")
        self.assertServerSettings('deadbeef', '7654')

    def test_no_database(self):
        """
        Ensure runserver.check_migrations doesn't choke on empty DATABASES.
        """
        tested_connections = ConnectionHandler({})
        with mock.patch('django.core.management.base.connections', new=tested_connections):
            self.cmd.check_migrations()

    def test_readonly_database(self):
        """
        Ensure runserver.check_migrations doesn't choke when a database is read-only
        (with possibly no django_migrations table).
        """
        with mock.patch.object(
                MigrationRecorder, 'ensure_schema',
                side_effect=MigrationSchemaMissing()):
            self.cmd.check_migrations()
        # Check a warning is emitted
        self.assertIn("Not checking migrations", self.output.getvalue())


class ManageRunserverMigrationWarning(TestCase):

    def setUp(self):
        from django.core.management.commands.runserver import Command
        self.stdout = StringIO()
        self.runserver_command = Command(stdout=self.stdout)

    @override_settings(INSTALLED_APPS=["admin_scripts.app_waiting_migration"])
    def test_migration_warning_one_app(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 1 unapplied migration(s)', output)
        self.assertIn('apply the migrations for app(s): app_waiting_migration.', output)

    @override_settings(
        INSTALLED_APPS=[
            "admin_scripts.app_waiting_migration",
            "admin_scripts.another_app_waiting_migration",
        ],
    )
    def test_migration_warning_multiple_apps(self):
        self.runserver_command.check_migrations()
        output = self.stdout.getvalue()
        self.assertIn('You have 2 unapplied migration(s)', output)
        self.assertIn(
            'apply the migrations for app(s): another_app_waiting_migration, '
            'app_waiting_migration.', output
        )


class ManageRunserverEmptyAllowedHosts(AdminScriptTestCase):
    def setUp(self):
        self.write_settings('settings.py', sdict={
            'ALLOWED_HOSTS': [],
            'DEBUG': False,
        })

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_empty_allowed_hosts_error(self):
        out, err = self.run_manage(['runserver'])
        self.assertNoOutput(out)
        self.assertOutput(err, 'CommandError: You must set settings.ALLOWED_HOSTS if DEBUG is False.')


class ManageTestserver(AdminScriptTestCase):
    from django.core.management.commands.testserver import Command as TestserverCommand

    @mock.patch.object(TestserverCommand, 'handle')
    def test_testserver_handle_params(self, mock_handle):
        out = StringIO()
        call_command('testserver', 'blah.json', stdout=out)
        mock_handle.assert_called_with(
            'blah.json',
            stdout=out, settings=None, pythonpath=None, verbosity=1,
            traceback=False, addrport='', no_color=False, use_ipv6=False,
            skip_checks=True, interactive=True,
        )


##########################################################################
# COMMAND PROCESSING TESTS
# Check that user-space commands are correctly handled - in particular,
# that arguments to the commands are correctly parsed and processed.
##########################################################################

class CommandTypes(AdminScriptTestCase):
    "Tests for the various types of base command types that can be defined."
    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_version(self):
        "version is handled as a special case"
        args = ['version']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, get_version())

    def test_version_alternative(self):
        "--version is equivalent to version"
        args1, args2 = ['version'], ['--version']
        # It's possible one outputs on stderr and the other on stdout, hence the set
        self.assertEqual(set(self.run_manage(args1)), set(self.run_manage(args2)))

    def test_help(self):
        "help is handled as a special case"
        args = ['help']
        out, err = self.run_manage(args)
        self.assertOutput(out, "Type 'manage.py help <subcommand>' for help on a specific subcommand.")
        self.assertOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')

    def test_help_commands(self):
        "help --commands shows the list of all available commands"
        args = ['help', '--commands']
        out, err = self.run_manage(args)
        self.assertNotInOutput(out, 'usage:')
        self.assertNotInOutput(out, 'Options:')
        self.assertNotInOutput(out, '[django]')
        self.assertOutput(out, 'startapp')
        self.assertOutput(out, 'startproject')
        self.assertNotInOutput(out, '\n\n')

    def test_help_alternative(self):
        "--help is equivalent to help"
        args1, args2 = ['help'], ['--help']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_help_short_altert(self):
        "-h is handled as a short form of --help"
        args1, args2 = ['--help'], ['-h']
        self.assertEqual(self.run_manage(args1), self.run_manage(args2))

    def test_specific_help(self):
        "--help can be used on a specific command"
        args = ['check', '--help']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "Checks the entire Django project for potential problems.")

    def test_color_style(self):
        style = color.no_style()
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('nocolor')
        self.assertEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        style = color.make_style('dark')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

        # Default palette has color.
        style = color.make_style('')
        self.assertIn('Hello, world!', style.ERROR('Hello, world!'))
        self.assertNotEqual(style.ERROR('Hello, world!'), 'Hello, world!')

    def test_command_color(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write('Hello, world!', self.style.ERROR)
                self.stderr.write('Hello, world!', self.style.ERROR)

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err)
        call_command(command)
        if color.supports_color():
            self.assertIn('Hello, world!\n', out.getvalue())
            self.assertIn('Hello, world!\n', err.getvalue())
            self.assertNotEqual(out.getvalue(), 'Hello, world!\n')
            self.assertNotEqual(err.getvalue(), 'Hello, world!\n')
        else:
            self.assertEqual(out.getvalue(), 'Hello, world!\n')
            self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_command_no_color(self):
        "--no-color prevent colorization of the output"
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write('Hello, world!', self.style.ERROR)
                self.stderr.write('Hello, world!', self.style.ERROR)

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err, no_color=True)
        call_command(command)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

        out = StringIO()
        err = StringIO()
        command = Command(stdout=out, stderr=err)
        call_command(command, no_color=True)
        self.assertEqual(out.getvalue(), 'Hello, world!\n')
        self.assertEqual(err.getvalue(), 'Hello, world!\n')

    def test_custom_stdout(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stdout.write("Hello, World!")

        out = StringIO()
        command = Command(stdout=out)
        call_command(command)
        self.assertEqual(out.getvalue(), "Hello, World!\n")
        out.truncate(0)
        new_out = StringIO()
        call_command(command, stdout=new_out)
        self.assertEqual(out.getvalue(), "")
        self.assertEqual(new_out.getvalue(), "Hello, World!\n")

    def test_custom_stderr(self):
        class Command(BaseCommand):
            requires_system_checks = False

            def handle(self, *args, **options):
                self.stderr.write("Hello, World!")

        err = StringIO()
        command = Command(stderr=err)
        call_command(command)
        self.assertEqual(err.getvalue(), "Hello, World!\n")
        err.truncate(0)
        new_err = StringIO()
        call_command(command, stderr=new_err)
        self.assertEqual(err.getvalue(), "")
        self.assertEqual(new_err.getvalue(), "Hello, World!\n")

    def test_base_command(self):
        "User BaseCommands can execute when a label is provided"
        args = ['base_command', 'testlabel']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels)

    def test_base_command_no_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command']
        expected_labels = "()"
        self._test_base_command(args, expected_labels)

    def test_base_command_multiple_label(self):
        "User BaseCommands can execute when no labels are provided"
        args = ['base_command', 'testlabel', 'anotherlabel']
        expected_labels = "('testlabel', 'anotherlabel')"
        self._test_base_command(args, expected_labels)

    def test_base_command_with_option(self):
        "User BaseCommands can execute with options when a label is provided"
        args = ['base_command', 'testlabel', '--option_a=x']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'")

    def test_base_command_with_options(self):
        "User BaseCommands can execute with multiple options when a label is provided"
        args = ['base_command', 'testlabel', '-a', 'x', '--option_b=y']
        expected_labels = "('testlabel',)"
        self._test_base_command(args, expected_labels, option_a="'x'", option_b="'y'")

    def test_base_command_with_wrong_option(self):
        "User BaseCommands outputs command usage when wrong option is specified"
        args = ['base_command', '--invalid']
        out, err = self.run_manage(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "usage: manage.py base_command")
        self.assertOutput(err, "error: unrecognized arguments: --invalid")

    def _test_base_command(self, args, labels, option_a="'1'", option_b="'2'"):
        out, err = self.run_manage(args)

        expected_out = (
            "EXECUTE:BaseCommand labels=%s, "
            "options=[('no_color', False), ('option_a', %s), ('option_b', %s), "
            "('option_c', '3'), ('pythonpath', None), ('settings', None), "
            "('traceback', False), ('verbosity', 1)]") % (labels, option_a, option_b)
        self.assertNoOutput(err)
        self.assertOutput(out, expected_out)

    def test_base_run_from_argv(self):
        """
        Test run_from_argv properly terminates even with custom execute() (#19665)
        Also test proper traceback display.
        """
        err = StringIO()
        command = BaseCommand(stderr=err)

        def raise_command_error(*args, **kwargs):
            raise CommandError("Custom error")

        command.execute = lambda args: args  # This will trigger TypeError

        # If the Exception is not CommandError it should always
        # raise the original exception.
        with self.assertRaises(TypeError):
            command.run_from_argv(['', ''])

        # If the Exception is CommandError and --traceback is not present
        # this command should raise a SystemExit and don't print any
        # traceback to the stderr.
        command.execute = raise_command_error
        err.truncate(0)
        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])
        err_message = err.getvalue()
        self.assertNotIn("Traceback", err_message)
        self.assertIn("CommandError", err_message)

        # If the Exception is CommandError and --traceback is present
        # this command should raise the original CommandError as if it
        # were not a CommandError.
        err.truncate(0)
        with self.assertRaises(CommandError):
            command.run_from_argv(['', '', '--traceback'])

    def test_run_from_argv_non_ascii_error(self):
        """
        Test that non-ASCII message of CommandError does not raise any
        UnicodeDecodeError in run_from_argv.
        """
        def raise_command_error(*args, **kwargs):
            raise CommandError("Erreur personnalisée")

        command = BaseCommand(stderr=StringIO())
        command.execute = raise_command_error

        with self.assertRaises(SystemExit):
            command.run_from_argv(['', ''])

    def test_run_from_argv_closes_connections(self):
        """
        A command called from the command line should close connections after
        being executed (#21255).
        """
        command = BaseCommand(stderr=StringIO())
        command.check = lambda: []
        command.handle = lambda *args, **kwargs: args
        with mock.patch('django.core.management.base.connections') as mock_connections:
            command.run_from_argv(['', ''])
        # Test connections have been closed
        self.assertTrue(mock_connections.close_all.called)

    def test_noargs(self):
        "NoArg Commands can be executed"
        args = ['noargs_command']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE: noargs_command options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), "
            "('verbosity', 1)]"
        )

    def test_noargs_with_args(self):
        "NoArg Commands raise an error if an argument is provided"
        args = ['noargs_command', 'argument']
        out, err = self.run_manage(args)
        self.assertOutput(err, "error: unrecognized arguments: argument")

    def test_app_command(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'auth']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_app_command_no_apps(self):
        "User AppCommands raise an error when no app name is provided"
        args = ['app_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'error: Enter at least one application label.')

    def test_app_command_multiple_apps(self):
        "User AppCommands raise an error when multiple app names are provided"
        args = ['app_command', 'auth', 'contenttypes']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.auth, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertOutput(out, "EXECUTE:AppCommand name=django.contrib.contenttypes, options=")
        self.assertOutput(
            out,
            ", options=[('no_color', False), ('pythonpath', None), "
            "('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_app_command_invalid_app_label(self):
        "User AppCommands can execute when a single app name is provided"
        args = ['app_command', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_app_command_some_invalid_app_labels(self):
        "User AppCommands can execute when some of the provided app names are invalid"
        args = ['app_command', 'auth', 'NOT_AN_APP']
        out, err = self.run_manage(args)
        self.assertOutput(err, "No installed app with label 'NOT_AN_APP'.")

    def test_label_command(self):
        "User LabelCommands can execute when a label is provided"
        args = ['label_command', 'testlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )

    def test_label_command_no_label(self):
        "User LabelCommands raise an error if no label is provided"
        args = ['label_command']
        out, err = self.run_manage(args)
        self.assertOutput(err, 'Enter at least one label')

    def test_label_command_multiple_label(self):
        "User LabelCommands are executed multiple times if multiple labels are provided"
        args = ['label_command', 'testlabel', 'anotherlabel']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=testlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )
        self.assertOutput(
            out,
            "EXECUTE:LabelCommand label=anotherlabel, options=[('no_color', False), "
            "('pythonpath', None), ('settings', None), ('traceback', False), ('verbosity', 1)]"
        )


class Discovery(SimpleTestCase):

    def test_precedence(self):
        """
        Apps listed first in INSTALLED_APPS have precedence.
        """
        with self.settings(INSTALLED_APPS=['admin_scripts.complex_app',
                                           'admin_scripts.simple_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'complex_app')
        with self.settings(INSTALLED_APPS=['admin_scripts.simple_app',
                                           'admin_scripts.complex_app',
                                           'django.contrib.auth',
                                           'django.contrib.contenttypes']):
            out = StringIO()
            call_command('duplicate', stdout=out)
            self.assertEqual(out.getvalue().strip(), 'simple_app')


class ArgumentOrder(AdminScriptTestCase):
    """Tests for 2-stage argument parsing scheme.

    django-admin command arguments are parsed in 2 parts; the core arguments
    (--settings, --traceback and --pythonpath) are parsed using a basic parser,
    ignoring any unknown options. Then the full settings are
    passed to the command parser, which extracts commands of interest to the
    individual command.
    """
    def setUp(self):
        self.write_settings('settings.py', apps=['django.contrib.auth', 'django.contrib.contenttypes'])
        self.write_settings('alternate_settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')
        self.remove_settings('alternate_settings.py')

    def test_setting_then_option(self):
        """ Options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '--option_a=x']
        self._test(args)

    def test_setting_then_short_option(self):
        """ Short options passed after settings are correctly handled. """
        args = ['base_command', 'testlabel', '--settings=alternate_settings', '-a', 'x']
        self._test(args)

    def test_option_then_setting(self):
        """ Options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings']
        self._test(args)

    def test_short_option_then_setting(self):
        """ Short options passed before settings are correctly handled. """
        args = ['base_command', 'testlabel', '-a', 'x', '--settings=alternate_settings']
        self._test(args)

    def test_option_then_setting_then_option(self):
        """ Options are correctly handled when they are passed before and after
        a setting. """
        args = ['base_command', 'testlabel', '--option_a=x', '--settings=alternate_settings', '--option_b=y']
        self._test(args, option_b="'y'")

    def _test(self, args, option_b="'2'"):
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(
            out,
            "EXECUTE:BaseCommand labels=('testlabel',), options=[('no_color', False), "
            "('option_a', 'x'), ('option_b', %s), ('option_c', '3'), "
            "('pythonpath', None), ('settings', 'alternate_settings'), "
            "('traceback', False), ('verbosity', 1)]" % option_b
        )


@override_settings(ROOT_URLCONF='admin_scripts.urls')
class StartProject(LiveServerTestCase, AdminScriptTestCase):

    available_apps = [
        'admin_scripts',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
    ]

    def test_wrong_args(self):
        "Make sure passing the wrong kinds of arguments outputs an error and prints usage"
        out, err = self.run_django_admin(['startproject'])
        self.assertNoOutput(out)
        self.assertOutput(err, "usage:")
        self.assertOutput(err, "You must provide a project name.")

    def test_simple_project(self):
        "Make sure the startproject management command creates a project"
        args = ['startproject', 'testproject']
        testproject_dir = os.path.join(self.test_dir, 'testproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_invalid_project_name(self):
        "Make sure the startproject management command validates a project name"
        for bad_name in ('7testproject', '../testproject'):
            args = ['startproject', bad_name]
            testproject_dir = os.path.join(self.test_dir, bad_name)
            self.addCleanup(shutil.rmtree, testproject_dir, True)

            out, err = self.run_django_admin(args)
            if PY2:
                self.assertOutput(
                    err,
                    "Error: '%s' is not a valid project name. Please make "
                    "sure the name begins with a letter or underscore." % bad_name
                )
            else:
                self.assertOutput(
                    err,
                    "Error: '%s' is not a valid project name. Please make "
                    "sure the name is a valid identifier." % bad_name
                )
            self.assertFalse(os.path.exists(testproject_dir))

    def test_simple_project_different_directory(self):
        "Make sure the startproject management command creates a project in a specific directory"
        args = ['startproject', 'testproject', 'othertestproject']
        testproject_dir = os.path.join(self.test_dir, 'othertestproject')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'manage.py')))

        # running again..
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "already exists")

    def test_custom_project_template(self):
        "Make sure the startproject management command is able to use a different project template"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_template_dir_with_trailing_slash(self):
        "Ticket 17475: Template dir passed has a trailing path separator"
        template_path = os.path.join(custom_templates_dir, 'project_template' + os.sep)
        args = ['startproject', '--template', template_path, 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))

    def test_custom_project_template_from_tarball_by_path(self):
        "Make sure the startproject management command is able to use a different project template from a tarball"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject']
        testproject_dir = os.path.join(self.test_dir, 'tarballtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_to_alternative_location(self):
        "Startproject can use a project template from a tarball and create it in a specified location"
        template_path = os.path.join(custom_templates_dir, 'project_template.tgz')
        args = ['startproject', '--template', template_path, 'tarballtestproject', 'altlocation']
        testproject_dir = os.path.join(self.test_dir, 'altlocation')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_custom_project_template_from_tarball_by_url(self):
        """
        The startproject management command is able to use a different project
        template from a tarball via a URL.
        """
        template_url = '%s/custom_templates/project_template.tgz' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_project_template_tarball_url(self):
        "Startproject management command handles project template tar/zip balls from non-canonical urls"
        template_url = '%s/custom_templates/project_template.tgz/' % self.live_server_url

        args = ['startproject', '--template', template_url, 'urltestproject']
        testproject_dir = os.path.join(self.test_dir, 'urltestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'run.py')))

    def test_file_without_extension(self):
        "Make sure the startproject management command is able to render custom files"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'customtestproject', '-e', 'txt', '-n', 'Procfile']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        self.assertTrue(os.path.exists(os.path.join(testproject_dir, 'additional_dir')))
        base_path = os.path.join(testproject_dir, 'additional_dir')
        for f in ('Procfile', 'additional_file.py', 'requirements.txt'):
            self.assertTrue(os.path.exists(os.path.join(base_path, f)))
            with open(os.path.join(base_path, f)) as fh:
                self.assertEqual(fh.read().strip(),
                    '# some file for customtestproject test project')

    def test_custom_project_template_context_variables(self):
        "Make sure template context variables are rendered with proper values"
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'another_project', 'project_dir']
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'manage.py')
        with open(test_manage_py, 'r') as fp:
            content = force_text(fp.read())
            self.assertIn("project_name = 'another_project'", content)
            self.assertIn("project_directory = '%s'" % testproject_dir, content)

    def test_no_escaping_of_project_variables(self):
        "Make sure template context variables are not html escaped"
        # We're using a custom command so we need the alternate settings
        self.write_settings('alternate_settings.py')
        self.addCleanup(self.remove_settings, 'alternate_settings.py')
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = [
            'custom_startproject', '--template', template_path,
            'another_project', 'project_dir', '--extra', '<&>',
            '--settings=alternate_settings',
        ]
        testproject_dir = os.path.join(self.test_dir, 'project_dir')
        os.mkdir(testproject_dir)
        self.addCleanup(shutil.rmtree, testproject_dir)
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        test_manage_py = os.path.join(testproject_dir, 'additional_dir', 'extra.py')
        with open(test_manage_py, 'r') as fp:
            content = fp.read()
            self.assertIn("<&>", content)

    def test_custom_project_destination_missing(self):
        """
        Make sure an exception is raised when the provided
        destination directory doesn't exist
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, 'yet_another_project', 'project_dir2']
        testproject_dir = os.path.join(self.test_dir, 'project_dir2')
        out, err = self.run_django_admin(args)
        self.assertNoOutput(out)
        self.assertOutput(err, "Destination directory '%s' does not exist, please create it first." % testproject_dir)
        self.assertFalse(os.path.exists(testproject_dir))

    def test_custom_project_template_with_non_ascii_templates(self):
        """
        The startproject management command is able to render templates with
        non-ASCII content.
        """
        template_path = os.path.join(custom_templates_dir, 'project_template')
        args = ['startproject', '--template', template_path, '--extension=txt', 'customtestproject']
        testproject_dir = os.path.join(self.test_dir, 'customtestproject')
        self.addCleanup(shutil.rmtree, testproject_dir, True)

        out, err = self.run_django_admin(args)
        self.assertNoOutput(err)
        self.assertTrue(os.path.isdir(testproject_dir))
        path = os.path.join(testproject_dir, 'ticket-18091-non-ascii-template.txt')
        with codecs.open(path, 'r', encoding='utf-8') as f:
            self.assertEqual(f.read().splitlines(False), [
                'Some non-ASCII text for testing ticket #18091:',
                'üäö €'])


class DiffSettings(AdminScriptTestCase):
    """Tests for diffsettings management command."""

    def test_basic(self):
        """Runs without error and emits settings diff."""
        self.write_settings('settings_to_diff.py', sdict={'FOO': '"bar"'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "FOO = 'bar'  ###")

    def test_all(self):
        """The all option also shows settings with the default value."""
        self.write_settings('settings_to_diff.py', sdict={'STATIC_URL': 'None'})
        self.addCleanup(self.remove_settings, 'settings_to_diff.py')
        args = ['diffsettings', '--settings=settings_to_diff', '--all']
        out, err = self.run_manage(args)
        self.assertNoOutput(err)
        self.assertOutput(out, "### STATIC_URL = None")


class Dumpdata(AdminScriptTestCase):
    """Tests for dumpdata management command."""

    def setUp(self):
        self.write_settings('settings.py')

    def tearDown(self):
        self.remove_settings('settings.py')

    def test_pks_parsing(self):
        """Regression for #20509

        Test would raise an exception rather than printing an error message.
        """
        args = ['dumpdata', '--pks=1']
        out, err = self.run_manage(args)
        self.assertOutput(err, "You can only use --pks option with one model")
        self.assertNoOutput(out)


class MainModule(AdminScriptTestCase):
    """python -m django works like django-admin."""

    def test_runs_django_admin(self):
        cmd_out, _ = self.run_django_admin(['--version'])
        mod_out, _ = self.run_test('-m', ['django', '--version'])
        self.assertEqual(mod_out, cmd_out)