refactor: Remove safer_staticfiles, replace STATICFILES_DIRS with COMPONENTS.dirs, support [app]/components (#652)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/django_components/app_settings.py

@@ -1,5 +1,6 @@
+import re
 from enum import Enum
-from typing import TYPE_CHECKING, Dict, List, Union
+from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
 from django.conf import settings
 
@@ -98,6 +99,14 @@ class AppSettings:
     def AUTODISCOVER(self) -> bool:
         return self.settings.get("autodiscover", True)
 
+    @property
+    def DIRS(self) -> List[Union[str, Tuple[str, str]]]:
+        return self.settings.get("dirs", [settings.BASE_DIR / "components"])
+
+    @property
+    def APP_DIRS(self) -> List[str]:
+        return self.settings.get("app_dirs", ["components"])
+
     @property
     def DYNAMIC_COMPONENT_NAME(self) -> str:
         return self.settings.get("dynamic_component_name", "dynamic")
@@ -118,6 +127,51 @@ class AppSettings:
     def TEMPLATE_CACHE_SIZE(self) -> int:
         return self.settings.get("template_cache_size", 128)
 
+    @property
+    def STATIC_FILES_ALLOWED(self) -> List[Union[str, re.Pattern]]:
+        default_static_files = [
+            ".css",
+            ".js",
+            # Images - See https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types#common_image_file_types  # noqa: E501
+            ".apng",
+            ".png",
+            ".avif",
+            ".gif",
+            ".jpg",
+            ".jpeg",
+            ".jfif",
+            ".pjpeg",
+            ".pjp",
+            ".svg",
+            ".webp",
+            ".bmp",
+            ".ico",
+            ".cur",
+            ".tif",
+            ".tiff",
+            # Fonts - See https://stackoverflow.com/q/30572159/9788634
+            ".eot",
+            ".ttf",
+            ".woff",
+            ".otf",
+            ".svg",
+        ]
+        return self.settings.get("static_files_allowed", default_static_files)
+
+    @property
+    def STATIC_FILES_FORBIDDEN(self) -> List[Union[str, re.Pattern]]:
+        default_forbidden_static_files = [
+            ".html",
+            # See https://marketplace.visualstudio.com/items?itemName=junstyle.vscode-django-support
+            ".django",
+            ".dj",
+            ".tpl",
+            # Python files
+            ".py",
+            ".pyc",
+        ]
+        return self.settings.get("forbidden_static_files", default_forbidden_static_files)
+
     @property
     def CONTEXT_BEHAVIOR(self) -> ContextBehavior:
         raw_value = self.settings.get("context_behavior", ContextBehavior.DJANGO.value)

src/django_components/autodiscover.py

@@ -5,10 +5,9 @@ from pathlib import Path
 from typing import Callable, List, Optional, Union
 
 from django.conf import settings
-from django.template.engine import Engine
 
 from django_components.logger import logger
-from django_components.template_loader import Loader
+from django_components.template_loader import get_dirs
 
 
 def autodiscover(
@@ -27,7 +26,13 @@ def autodiscover(
     component_filepaths = search_dirs(dirs, "**/*.py")
     logger.debug(f"Autodiscover found {len(component_filepaths)} files in component directories.")
 
-    modules = [_filepath_to_python_module(filepath) for filepath in component_filepaths]
+    modules: List[str] = []
+    for filepath in component_filepaths:
+        module_path = _filepath_to_python_module(filepath)
+        # Ignore relative paths that are outside of the project root
+        if not module_path.startswith(".."):
+            modules.append(module_path)
+
     return _import_modules(modules, map_module)
 
 
@@ -89,19 +94,6 @@ def _filepath_to_python_module(file_path: Union[Path, str]) -> str:
     return module_name
 
 
-def get_dirs(engine: Optional[Engine] = None) -> List[Path]:
-    """
-    Helper for using django_component's FilesystemLoader class to obtain a list
-    of directories where component python files may be defined.
-    """
-    current_engine = engine
-    if current_engine is None:
-        current_engine = Engine.get_default()
-
-    loader = Loader(current_engine)
-    return loader.get_dirs()
-
-
 def search_dirs(dirs: List[Path], search_glob: str) -> List[Path]:
     """
     Search the directories for the given glob pattern. Glob search results are returned

src/django_components/component_media.py

@@ -284,7 +284,7 @@ def _resolve_component_relative_files(attrs: MutableMapping) -> None:
             f"No component directory found for component '{component_name}' in {file_path}"
             " If this component defines HTML, JS or CSS templates relatively to the component file,"
             " then check that the component's directory is accessible from one of the paths"
-            " specified in the Django's 'STATICFILES_DIRS' settings."
+            " specified in the Django's 'COMPONENTS.dirs' settings."
         )
         return
 
@@ -327,7 +327,7 @@ def _get_dir_path_from_component_path(
 ) -> Tuple[str, str]:
     comp_dir_path_abs = os.path.dirname(abs_component_file_path)
 
-    # From all dirs defined in settings.STATICFILES_DIRS, find one that's the parent
+    # From all dirs defined in settings.COMPONENTS.dirs, find one that's the parent
     # to the component file.
     root_dir_abs = None
     for candidate_dir in candidate_dirs:
@@ -341,7 +341,7 @@ def _get_dir_path_from_component_path(
             f"Failed to resolve template directory for component file '{abs_component_file_path}'",
         )
 
-    # Derive the path from matched STATICFILES_DIRS to the dir where the current component file is.
+    # Derive the path from matched COMPONENTS.dirs to the dir where the current component file is.
     comp_dir_path_rel = os.path.relpath(comp_dir_path_abs, candidate_dir_abs)
 
     # Return both absolute and relative paths:

src/django_components/finders.py

@@ -0,0 +1,154 @@
+import os
+import re
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+
+from django.contrib.staticfiles.finders import BaseFinder
+from django.contrib.staticfiles.utils import get_files
+from django.core.checks import CheckMessage, Error, Warning
+from django.core.files.storage import FileSystemStorage
+from django.utils._os import safe_join
+
+from django_components.app_settings import app_settings
+from django_components.template_loader import get_dirs
+from django_components.utils import any_regex_match, no_regex_match
+
+# To keep track on which directories the finder has searched the static files.
+searched_locations = []
+
+
+# Custom Finder for staticfiles that searches for all files within the directories
+# defined by `COMPONENTS.dirs`.
+#
+# This is what makes it possible to define JS and CSS files in the directories as
+# defined by `COMPONENTS.dirs`, but still use the JS / CSS files with `static()` or
+# `collectstatic` command.
+class ComponentsFileSystemFinder(BaseFinder):
+    """
+    A static files finder based on `FileSystemFinder`.
+
+    Differences:
+    - This finder uses `COMPONENTS.dirs` setting to locate files instead of `STATICFILES_DIRS`.
+    - Whether a file within `COMPONENTS.dirs` is considered a STATIC file is configured
+      by `COMPONENTS.static_files_allowed` and `COMPONENTS.forbidden_static_files`.
+    - If `COMPONENTS.dirs` is not set, defaults to `settings.BASE_DIR / "components"`
+    """
+
+    def __init__(self, app_names: Any = None, *args: Any, **kwargs: Any) -> None:
+        component_dirs = [str(p) for p in get_dirs()]
+
+        # NOTE: The rest of the __init__ is the same as `django.contrib.staticfiles.finders.FileSystemFinder`,
+        # but using our locations instead of STATICFILES_DIRS.
+
+        # List of locations with static files
+        self.locations: List[Tuple[str, str]] = []
+
+        # Maps dir paths to an appropriate storage instance
+        self.storages: Dict[str, FileSystemStorage] = {}
+        for root in component_dirs:
+            if isinstance(root, (list, tuple)):
+                prefix, root = root
+            else:
+                prefix = ""
+            if (prefix, root) not in self.locations:
+                self.locations.append((prefix, root))
+        for prefix, root in self.locations:
+            filesystem_storage = FileSystemStorage(location=root)
+            filesystem_storage.prefix = prefix
+            self.storages[root] = filesystem_storage
+
+        super().__init__(*args, **kwargs)
+
+    # NOTE: Based on `FileSystemFinder.check`
+    def check(self, **kwargs: Any) -> List[CheckMessage]:
+        errors: List[CheckMessage] = []
+        if not isinstance(app_settings.DIRS, (list, tuple)):
+            errors.append(
+                Error(
+                    "The COMPONENTS.dirs setting is not a tuple or list.",
+                    hint="Perhaps you forgot a trailing comma?",
+                    id="components.E001",
+                )
+            )
+            return errors
+        for root in app_settings.DIRS:
+            if isinstance(root, (list, tuple)):
+                prefix, root = root
+                if prefix.endswith("/"):
+                    errors.append(
+                        Error(
+                            "The prefix %r in the COMPONENTS.dirs setting must not end with a slash." % prefix,
+                            id="staticfiles.E003",
+                        )
+                    )
+            elif not os.path.isdir(root):
+                errors.append(
+                    Warning(
+                        f"The directory '{root}' in the COMPONENTS.dirs setting does not exist.",
+                        id="components.W004",
+                    )
+                )
+        return errors
+
+    # NOTE: Same as `FileSystemFinder.find`
+    def find(self, path: str, all: bool = False) -> Union[List[str], str]:
+        """
+        Look for files in the extra locations as defined in COMPONENTS.dirs.
+        """
+        matches: List[str] = []
+        for prefix, root in self.locations:
+            if root not in searched_locations:
+                searched_locations.append(root)
+            matched_path = self.find_location(root, path, prefix)
+            if matched_path:
+                if not all:
+                    return matched_path
+                matches.append(matched_path)
+        return matches
+
+    # NOTE: Same as `FileSystemFinder.find_local`, but we exclude Python/HTML files
+    def find_location(self, root: str, path: str, prefix: Optional[str] = None) -> Optional[str]:
+        """
+        Find a requested static file in a location and return the found
+        absolute path (or ``None`` if no match).
+        """
+        if prefix:
+            prefix = "%s%s" % (prefix, os.sep)
+            if not path.startswith(prefix):
+                return None
+            path = path.removeprefix(prefix)
+        path = safe_join(root, path)
+
+        if os.path.exists(path) and self._is_path_valid(path):
+            return path
+        return None
+
+    # `Finder.list` is called from `collectstatic` command,
+    # see https://github.com/django/django/blob/bc9b6251e0b54c3b5520e3c66578041cc17e4a28/django/contrib/staticfiles/management/commands/collectstatic.py#L126C23-L126C30  # noqa E501
+    #
+    # NOTE: This is same as `FileSystemFinder.list`, but we exclude Python/HTML files
+    # NOTE 2: Yield can be annotated as Iterable, see https://stackoverflow.com/questions/38419654
+    def list(self, ignore_patterns: List[str]) -> Iterable[Tuple[str, FileSystemStorage]]:
+        """
+        List all files in all locations.
+        """
+        for prefix, root in self.locations:
+            # Skip nonexistent directories.
+            if os.path.isdir(root):
+                storage = self.storages[root]
+                for path in get_files(storage, ignore_patterns):
+                    if self._is_path_valid(path):
+                        yield path, storage
+
+    def _is_path_valid(self, path: str) -> bool:
+        # Normalize patterns to regexes
+        allowed_patterns = [
+            # Convert suffixes like `.html` to regex `\.html$`
+            re.compile(rf"\{p}$") if isinstance(p, str) else p
+            for p in app_settings.STATIC_FILES_ALLOWED
+        ]
+        forbidden_patterns = [
+            # Convert suffixes like `.html` to regex `\.html$`
+            re.compile(rf"\{p}$") if isinstance(p, str) else p
+            for p in app_settings.STATIC_FILES_FORBIDDEN
+        ]
+        return any_regex_match(path, allowed_patterns) and no_regex_match(path, forbidden_patterns)

src/django_components/safer_staticfiles/apps.py

@@ -1,22 +0,0 @@
-from django.contrib.staticfiles.apps import StaticFilesConfig
-
-
-class SaferStaticFilesConfig(StaticFilesConfig):
-    """
-    Extend the `ignore_patterns` class attr of StaticFilesConfig to include Python
-    modules and HTML files.
-
-    When this class is registered as an installed app,
-    `$ ./manage.py collectstatic` will ignore .py and .html files,
-    preventing potentially sensitive backend logic from being leaked
-    by the static file server.
-
-    See https://docs.djangoproject.com/en/5.0/ref/contrib/staticfiles/#customizing-the-ignored-pattern-list
-    """
-
-    default = True  # Ensure that _this_ app is registered, as opposed to parent cls.
-    ignore_patterns = StaticFilesConfig.ignore_patterns + [
-        "*.py",
-        "*.html",
-        "*.pyc",
-    ]

src/django_components/template_loader.py

@@ -3,55 +3,92 @@ Template loader that loads templates from each Django app's "components" directo
 """
 
 from pathlib import Path
-from typing import List, Set
+from typing import List, Optional, Set
 
+from django.apps import apps
 from django.conf import settings
+from django.template.engine import Engine
 from django.template.loaders.filesystem import Loader as FilesystemLoader
 
+from django_components.app_settings import app_settings
 from django_components.logger import logger
 
 
+# Similar to `Path.is_relative_to`, which is missing in 3.8
+def is_relative_to(path: Path, other: Path) -> bool:
+    try:
+        path.relative_to(other)
+        return True
+    except ValueError:
+        return False
+
+
+# This is the heart of all features that deal with filesystem and file lookup.
+# Autodiscovery, Django template resolution, static file resolution - They all
+# depend on this loader.
 class Loader(FilesystemLoader):
     def get_dirs(self) -> List[Path]:
         """
         Prepare directories that may contain component files:
 
-        Searches for dirs set in `STATICFILES_DIRS` settings. If none set, defaults to searching
-        for a "components" app. The dirs in `STATICFILES_DIRS` must be absolute paths.
+        Searches for dirs set in `COMPONENTS.dirs` settings. If none set, defaults to searching
+        for a "components" app. The dirs in `COMPONENTS.dirs` must be absolute paths.
+
+        In addition to that, also all apps are checked for `[app]/components` dirs.
 
         Paths are accepted only if they resolve to a directory.
         E.g. `/path/to/django_project/my_app/components/`.
 
-        If `STATICFILES_DIRS` is not set or empty, then `BASE_DIR` is required.
+        `BASE_DIR` setting is required.
         """
         # Allow to configure from settings which dirs should be checked for components
-        if hasattr(settings, "STATICFILES_DIRS") and settings.STATICFILES_DIRS:
-            component_dirs = settings.STATICFILES_DIRS
-        else:
-            component_dirs = [settings.BASE_DIR / "components"]
+        component_dirs = app_settings.DIRS
+
+        # TODO_REMOVE_IN_V1
+        is_legacy_paths = (
+            # Use value of `STATICFILES_DIRS` ONLY if `COMPONENT.dirs` not set
+            not getattr(settings, "COMPONENTS", {}).get("dirs", None) is not None
+            and hasattr(settings, "STATICFILES_DIRS")
+            and settings.STATICFILES_DIRS
+        )
+        if is_legacy_paths:
+            # NOTE: For STATICFILES_DIRS, we use the defaults even for empty list.
+            # We don't do this for COMPONENTS.dirs, so user can explicitly specify "NO dirs".
+            component_dirs = settings.STATICFILES_DIRS or [settings.BASE_DIR / "components"]
+        source = "STATICFILES_DIRS" if is_legacy_paths else "COMPONENTS.dirs"
 
         logger.debug(
             "Template loader will search for valid template dirs from following options:\n"
             + "\n".join([f" - {str(d)}" for d in component_dirs])
         )
 
-        directories: Set[Path] = set()
+        # Add `[app]/[APP_DIR]` to the directories. This is, by default `[app]/components`
+        app_paths: List[Path] = []
+        for conf in apps.get_app_configs():
+            for app_dir in app_settings.APP_DIRS:
+                comps_path = Path(conf.path).joinpath(app_dir)
+                if comps_path.exists() and is_relative_to(comps_path, settings.BASE_DIR):
+                    app_paths.append(comps_path)
+
+        directories: Set[Path] = set(app_paths)
+
+        # Validate and add other values from the config
         for component_dir in component_dirs:
             # Consider tuples for STATICFILES_DIRS (See #489)
             # See https://docs.djangoproject.com/en/5.0/ref/settings/#prefixes-optional
-            if isinstance(component_dir, (tuple, list)) and len(component_dir) == 2:
+            if isinstance(component_dir, (tuple, list)):
                 component_dir = component_dir[1]
             try:
                 Path(component_dir)
             except TypeError:
                 logger.warning(
-                    f"STATICFILES_DIRS expected str, bytes or os.PathLike object, or tuple/list of length 2. "
-                    f"See Django documentation. Got {type(component_dir)} : {component_dir}"
+                    f"{source} expected str, bytes or os.PathLike object, or tuple/list of length 2. "
+                    f"See Django documentation for STATICFILES_DIRS. Got {type(component_dir)} : {component_dir}"
                 )
                 continue
 
             if not Path(component_dir).is_absolute():
-                raise ValueError(f"STATICFILES_DIRS must contain absolute paths, got '{component_dir}'")
+                raise ValueError(f"{source} must contain absolute paths, got '{component_dir}'")
             else:
                 directories.add(Path(component_dir).resolve())
 
@@ -59,3 +96,16 @@ class Loader(FilesystemLoader):
             "Template loader matched following template dirs:\n" + "\n".join([f" - {str(d)}" for d in directories])
         )
         return list(directories)
+
+
+def get_dirs(engine: Optional[Engine] = None) -> List[Path]:
+    """
+    Helper for using django_component's FilesystemLoader class to obtain a list
+    of directories where component python files may be defined.
+    """
+    current_engine = engine
+    if current_engine is None:
+        current_engine = Engine.get_default()
+
+    loader = Loader(current_engine)
+    return loader.get_dirs()

src/django_components/utils.py

@@ -1,4 +1,5 @@
 import functools
+import re
 import sys
 import typing
 from pathlib import Path
@@ -211,3 +212,11 @@ def lazy_cache(
         return cast(TFunc, wrapper)
 
     return decorator
+
+
+def any_regex_match(string: str, patterns: List[re.Pattern]) -> bool:
+    return any(p.search(string) is not None for p in patterns)
+
+
+def no_regex_match(string: str, patterns: List[re.Pattern]) -> bool:
+    return all(p.search(string) is None for p in patterns)