From 849a869a6609b517e025f2a448546275b96cc2a9 Mon Sep 17 00:00:00 2001 From: Juro Oravec Date: Sun, 22 Sep 2024 16:42:41 +0200 Subject: [PATCH] feat: add JS dependency manager (#666) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/publish-to-pypi.yml | 2 +- .github/workflows/tests.yml | 5 +- .gitignore | 5 + MANIFEST.in | 5 + README.md | 70 ++- requirements-ci.in | 6 + requirements-ci.txt | 58 +++ requirements-dev.in | 6 +- requirements-dev.txt | 28 +- .../django_components.min.js | 1 + src/django_components_js/README.md | 62 +++ src/django_components_js/build.py | 54 +++ src/django_components_js/package-lock.json | 435 +++++++++++++++++ src/django_components_js/package.json | 6 + src/django_components_js/src/errorHandling.ts | 45 ++ src/django_components_js/src/index.ts | 22 + src/django_components_js/src/manager.ts | 196 ++++++++ src/django_components_js/src/utils.ts | 15 + tests/conftest.py | 9 + tests/django_test_setup.py | 28 +- tests/e2e/__init__.py | 0 tests/e2e/testserver/__init__.py | 0 tests/e2e/testserver/manage.py | 22 + tests/e2e/testserver/testserver/__init__.py | 0 tests/e2e/testserver/testserver/asgi.py | 16 + tests/e2e/testserver/testserver/settings.py | 106 +++++ tests/e2e/testserver/testserver/urls.py | 3 + tests/e2e/testserver/testserver/wsgi.py | 16 + tests/e2e/utils.py | 71 +++ tests/test_dependency_manager.py | 442 ++++++++++++++++++ tox.ini | 15 +- 31 files changed, 1730 insertions(+), 19 deletions(-) create mode 100644 MANIFEST.in create mode 100644 requirements-ci.in create mode 100644 requirements-ci.txt create mode 100644 src/django_components/static/django_components/django_components.min.js create mode 100644 src/django_components_js/README.md create mode 100644 src/django_components_js/build.py create mode 100644 src/django_components_js/package-lock.json create mode 100644 src/django_components_js/package.json create mode 100644 src/django_components_js/src/errorHandling.ts create mode 100644 src/django_components_js/src/index.ts create mode 100644 src/django_components_js/src/manager.ts create mode 100644 src/django_components_js/src/utils.ts create mode 100644 tests/conftest.py create mode 100644 tests/e2e/__init__.py create mode 100644 tests/e2e/testserver/__init__.py create mode 100755 tests/e2e/testserver/manage.py create mode 100644 tests/e2e/testserver/testserver/__init__.py create mode 100644 tests/e2e/testserver/testserver/asgi.py create mode 100644 tests/e2e/testserver/testserver/settings.py create mode 100644 tests/e2e/testserver/testserver/urls.py create mode 100644 tests/e2e/testserver/testserver/wsgi.py create mode 100644 tests/e2e/utils.py create mode 100644 tests/test_dependency_manager.py diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 27a99d30..f89b2535 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout the repo uses: actions/checkout@v2 - + - name: Setup python uses: actions/setup-python@v2 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0b751024..c217fe95 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,6 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -24,6 +25,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install tox tox-gh-actions + python -m pip install -r requirements-ci.txt + # See https://playwright.dev/python/docs/intro#installing-playwright-pytest + playwright install chromium --with-deps - name: Run tests run: tox diff --git a/.gitignore b/.gitignore index 389be594..909e7fd2 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ coverage.xml # Django stuff: *.log +*.sqlite3 # Sphinx documentation docs/_build/ @@ -74,3 +75,7 @@ poetry.lock .python-version site docs/reference + +# JS, NPM Dependency directories +node_modules/ +jspm_packages/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..8b126249 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +# MANIFEST.in is defined so we can include non-Python (e.g. JS) files +# in the built distribution. +# See https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html +graft src/django_components/static +prune tests diff --git a/README.md b/README.md index f79f7248..fc2c7582 100644 --- a/README.md +++ b/README.md @@ -3852,7 +3852,7 @@ One of our goals with `django-components` is to make it easy to share components - [django-htmx-components](https://github.com/iwanalabs/django-htmx-components): A set of components for use with [htmx](https://htmx.org/). Try out the [live demo](https://dhc.iwanalabs.com/). -## Running django-components project locally +## Contributing and development ### Install locally and run the tests @@ -3886,6 +3886,22 @@ pyenv local 3.8 3.9 3.10 3.11 3.12 tox -p ``` +### Running Playwright tests + +We use [Playwright](https://playwright.dev/python/docs/intro) for end-to-end tests. You will therefore need to install Playwright to be able to run these tests. + +Luckily, Playwright makes it very easy: + +```sh +pip install -r requirements-dev.txt +playwright install chromium --with-deps +``` + +After Playwright is ready, simply run the tests with `tox`: +```sh +tox +``` + ### Developing against live Django app How do you check that your changes to django-components project will work in an actual Django project? @@ -3921,6 +3937,54 @@ Once the server is up, it should be available at . To display individual components, add them to the `urls.py`, like in the case of -## Development guides +### Building JS code -- [Slot rendering flot](https://github.com/EmilStenstrom/django-components/blob/master/docs/slot_rendering.md) \ No newline at end of file +django_components uses a bit of JS code to: +- Manage the loading of JS and CSS files used by the components +- Allow to pass data from Python to JS + +When you make changes to this JS code, you also need to compile it: + +1. Make sure you are inside `src/django_components_js`: + +```sh +cd src/django_components_js +``` + +2. Install the JS dependencies + +```sh +npm install +``` + +3. Compile the JS/TS code: + +```sh +python build.py +``` + +The script will combine all JS/TS code into a single `.js` file, minify it, +and copy it to `django_components/static/django_components/django_components.min.js`. + +### Packaging and publishing + +To package the library into a distribution that can be published to PyPI, run: + +```sh +# Install pypa/build +python -m pip install build --user +# Build a binary wheel and a source tarball +python -m build --sdist --wheel --outdir dist/ . +``` + +To publish the package to PyPI, use `twine` ([See Python user guide](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives)): +```sh +twine upload --repository pypi dist/* -u __token__ -p +``` + +[See the full workflow here.](https://github.com/EmilStenstrom/django-components/discussions/557#discussioncomment-10179141) + +### Development guides + +- [Slot rendering flot](https://github.com/EmilStenstrom/django-components/blob/master/docs/slot_rendering.md) +- [Slots and blocks](https://github.com/EmilStenstrom/django-components/blob/master/docs/slots_and_blocks.md) diff --git a/requirements-ci.in b/requirements-ci.in new file mode 100644 index 00000000..bf2f0500 --- /dev/null +++ b/requirements-ci.in @@ -0,0 +1,6 @@ +tox +tox-gh-actions +playwright +requests +types-requests +whitenoise \ No newline at end of file diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 00000000..b56276a1 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,58 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile requirements-ci.in +# +cachetools==5.5.0 + # via tox +certifi==2024.8.30 + # via requests +chardet==5.2.0 + # via tox +charset-normalizer==3.3.2 + # via requests +colorama==0.4.6 + # via tox +distlib==0.3.8 + # via virtualenv +filelock==3.16.1 + # via + # tox + # virtualenv +greenlet==3.0.3 + # via playwright +idna==3.10 + # via requests +packaging==24.1 + # via + # pyproject-api + # tox +platformdirs==4.3.6 + # via + # tox + # virtualenv +playwright==1.47.0 + # via -r requirements-ci.in +pluggy==1.5.0 + # via tox +pyee==12.0.0 + # via playwright +pyproject-api==1.8.0 + # via tox +requests==2.32.3 + # via -r requirements-ci.in +tox==4.20.0 + # via + # -r requirements-ci.in + # tox-gh-actions +tox-gh-actions==3.2.0 + # via -r requirements-ci.in +typing-extensions==4.12.2 + # via pyee +urllib3==2.2.3 + # via requests +virtualenv==20.26.5 + # via tox +whitenoise==6.7.0 + # via -r requirements-ci.in diff --git a/requirements-dev.in b/requirements-dev.in index 6ca102ac..c8f01700 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -6,4 +6,8 @@ flake8-pyproject isort pre-commit black -mypy \ No newline at end of file +mypy +playwright +requests +types-requests +whitenoise \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index 8e1dbcfa..8d83d26e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements-dev.in @@ -10,10 +10,14 @@ black==24.8.0 # via -r requirements-dev.in cachetools==5.4.0 # via tox +certifi==2024.8.30 + # via requests cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via black colorama==0.4.6 @@ -32,8 +36,12 @@ flake8==7.1.1 # flake8-pyproject flake8-pyproject==1.2.3 # via -r requirements-dev.in +greenlet==3.0.3 + # via playwright identify==2.5.33 # via pre-commit +idna==3.10 + # via requests iniconfig==2.0.0 # via pytest isort==5.13.2 @@ -61,6 +69,8 @@ platformdirs==4.2.2 # black # tox # virtualenv +playwright==1.47.0 + # via -r requirements-dev.in pluggy==1.5.0 # via # pytest @@ -69,6 +79,8 @@ pre-commit==3.8.0 # via -r requirements-dev.in pycodestyle==2.12.0 # via flake8 +pyee==12.0.0 + # via playwright pyflakes==3.2.0 # via flake8 pyproject-api==1.7.1 @@ -77,16 +89,28 @@ pytest==8.3.3 # via -r requirements-dev.in pyyaml==6.0.1 # via pre-commit +requests==2.32.3 + # via -r requirements-dev.in sqlparse==0.5.0 # via django tox==4.18.0 # via -r requirements-dev.in +types-requests==2.32.0.20240914 + # via -r requirements-dev.in typing-extensions==4.10.0 - # via mypy + # via + # mypy + # pyee +urllib3==2.2.3 + # via + # requests + # types-requests virtualenv==20.26.3 # via # pre-commit # tox +whitenoise==6.7.0 + # via -r requirements-dev.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/src/django_components/static/django_components/django_components.min.js b/src/django_components/static/django_components/django_components.min.js new file mode 100644 index 00000000..402af70c --- /dev/null +++ b/src/django_components/static/django_components/django_components.min.js @@ -0,0 +1 @@ +(()=>{var y=Array.isArray,p=t=>typeof t=="function",M=t=>t!==null&&typeof t=="object",h=t=>(M(t)||p(t))&&p(t.then)&&p(t.catch);function S(t,a){try{return a?t.apply(null,a):t()}catch(r){f(r)}}function m(t,a){if(p(t)){let r=S(t,a);return r&&h(r)&&r.catch(i=>{f(i)}),[r]}if(y(t)){let r=[];for(let i=0;i{let t=new Set,a=new Set,r={},i={},C=e=>{let n=new DOMParser().parseFromString(e,"text/html").querySelector("script");if(!n)throw Error("[Components] Failed to extract '); + * ``` + * + * ```js + * Components.markScriptLoaded("js", '/abc/def'); + * ``` + */ +export const createComponentsManager = () => { + const loadedJs = new Set(); + const loadedCss = new Set(); + const components: Record = {}; + const componentInputs: Record = {}; + + const parseScriptTag = (tag: string) => { + const scriptNode = new DOMParser().parseFromString(tag, 'text/html').querySelector('script'); + if (!scriptNode) { + throw Error( + '[Components] Failed to extract and is a valid HTML' + ); + } + return scriptNode; + }; + + const parseLinkTag = (tag: string) => { + const linkNode = new DOMParser().parseFromString(tag, 'text/html').querySelector('link'); + if (!linkNode) { + throw Error( + '[Components] Failed to extract tag. Make sure that the string contains' + + ' and is a valid HTML' + ); + } + return linkNode; + }; + + // NOTE: The way we turn the string into an HTMLElement, if we then try to + // insert the node into the Document, it will NOT load. So instead we create + // a "); + const bodyAfterFirstLoad = document.body.innerHTML; + + // Does not add it the second time + manager.loadScript('js', ""); + const bodyAfterSecondLoad = document.body.innerHTML; + + // Adds different script + manager.loadScript('js', ""); + const bodyAfterThirdLoad = document.body.innerHTML; + + const headAfterThirdLoad = document.head.innerHTML; + + return { + bodyAfterFirstLoad, + bodyAfterSecondLoad, + bodyAfterThirdLoad, + headBeforeFirstLoad, + headAfterThirdLoad, + }; + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data["bodyAfterFirstLoad"], '') + self.assertEqual(data["bodyAfterSecondLoad"], '') + self.assertEqual( + data["bodyAfterThirdLoad"], '' + ) + + self.assertEqual(data["headBeforeFirstLoad"], data["headAfterThirdLoad"]) + self.assertEqual(data["headBeforeFirstLoad"], "") + + await page.close() + + @with_playwright + async def test_load_css_scripts(self): + page = await self._create_page_with_dep_manager() + + # JS code that loads a few dependencies, capturing the HTML after each action + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const bodyBeforeFirstLoad = document.body.innerHTML; + + // Adds a script the first time + manager.loadScript('css', ""); + const headAfterFirstLoad = document.head.innerHTML; + + // Does not add it the second time + manager.loadScript('css', ""); + const headAfterSecondLoad = document.head.innerHTML; + + // Adds different script + manager.loadScript('css', ""); + const headAfterThirdLoad = document.head.innerHTML; + + const bodyAfterThirdLoad = document.body.innerHTML; + + return { + headAfterFirstLoad, + headAfterSecondLoad, + headAfterThirdLoad, + bodyBeforeFirstLoad, + bodyAfterThirdLoad, + }; + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data["headAfterFirstLoad"], '') + self.assertEqual(data["headAfterSecondLoad"], '') + self.assertEqual(data["headAfterThirdLoad"], '') + + self.assertEqual(data["bodyBeforeFirstLoad"], data["bodyAfterThirdLoad"]) + self.assertEqual(data["bodyBeforeFirstLoad"], "") + + await page.close() + + @with_playwright + async def test_does_not_load_script_if_marked_as_loaded(self): + page = await self._create_page_with_dep_manager() + + # JS code that loads a few dependencies, capturing the HTML after each action + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + // Adds a script the first time + manager.markScriptLoaded('css', '/one/two'); + manager.markScriptLoaded('js', '/one/three'); + + manager.loadScript('css', ""); + const headAfterFirstLoad = document.head.innerHTML; + + manager.loadScript('js', ""); + const bodyAfterSecondLoad = document.body.innerHTML; + + return { + headAfterFirstLoad, + bodyAfterSecondLoad, + }; + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data["headAfterFirstLoad"], "") + self.assertEqual(data["bodyAfterSecondLoad"], "") + + await page.close() + + +# Tests for `manager.registerComponent()` / `registerComponentData()` / `callComponent()` +@override_settings(STATIC_URL="static/") +class CallComponentTests(_BaseDepManagerTestCase): + @with_playwright + async def test_calls_component_successfully(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + // Pretend that this HTML belongs to our component + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + let captured = null; + manager.registerComponent(compName, (data, ctx) => { + captured = { ctx, data }; + return 123; + }); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + const result = manager.callComponent(compName, compId, inputHash); + + // Serialize the HTML elements + captured.ctx.els = captured.ctx.els.map((el) => el.outerHTML); + + return { + result, + captured, + }; + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data["result"], 123) + self.assertEqual( + data["captured"], + { + "data": { + "hello": "world", + }, + "ctx": { + "els": ['
abc
'], + "id": "12345", + "name": "my_comp", + }, + }, + ) + + await page.close() + + @with_playwright + async def test_calls_component_successfully_async(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + // Pretend that this HTML belongs to our component + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + manager.registerComponent(compName, (data, ctx) => { + return Promise.resolve(123); + }); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + // Should be Promise + const result = manager.callComponent(compName, compId, inputHash); + const isPromise = `${result}` === '[object Promise]'; + + // Wrap the whole response in Promise, so we can add extra fields + return Promise.resolve(result).then((res) => ({ + result: res, + isPromise, + })); + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data["result"], 123) + self.assertEqual(data["isPromise"], True) + + await page.close() + + @with_playwright + async def test_error_in_component_call_do_not_propagate_sync(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + // Pretend that this HTML belongs to our component + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + manager.registerComponent(compName, (data, ctx) => { + throw Error('Oops!'); + return 123; + }); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + const result = manager.callComponent(compName, compId, inputHash); + + return result; + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(data, None) + + await page.close() + + @with_playwright + async def test_error_in_component_call_do_not_propagate_async(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + // Pretend that this HTML belongs to our component + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + manager.registerComponent(compName, async (data, ctx) => { + throw Error('Oops!'); + return 123; + }); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + const result = manager.callComponent(compName, compId, inputHash); + return Promise.allSettled([result]); + }""" + + data = await page.evaluate(test_js) + + self.assertEqual(len(data), 1) + self.assertEqual(data[0]["status"], "rejected") + self.assertIsInstance(data[0]["reason"], Error) + self.assertEqual(data[0]["reason"].message, "Oops!") + + await page.close() + + @with_playwright + async def test_raises_if_component_element_not_in_dom(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + manager.registerComponent(compName, (data, ctx) => { + return 123; + }); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + // Should raise Error + manager.callComponent(compName, compId, inputHash); + }""" + + with self.assertRaisesMessage( + Error, "Error: [Components] 'my_comp': No elements with component ID '12345' found" + ): + await page.evaluate(test_js) + + await page.close() + + @with_playwright + async def test_raises_if_input_hash_not_registered(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + manager.registerComponent(compName, (data, ctx) => { + return Promise.resolve(123); + }); + + // Should raise Error + manager.callComponent(compName, compId, inputHash); + }""" + + with self.assertRaisesMessage(Error, "Error: [Components] 'my_comp': Cannot find input for hash 'input-abc'"): + await page.evaluate(test_js) + + await page.close() + + @with_playwright + async def test_raises_if_component_not_registered(self): + page = await self._create_page_with_dep_manager() + + test_js: types.js = """() => { + const manager = Components.createComponentsManager(); + + const compName = 'my_comp'; + const compId = '12345'; + const inputHash = 'input-abc'; + + document.body.insertAdjacentHTML('beforeend', '
abc
'); + + manager.registerComponentData(compName, inputHash, () => { + return { hello: 'world' }; + }); + + // Should raise Error + manager.callComponent(compName, compId, inputHash); + }""" + + with self.assertRaisesMessage(Error, "Error: [Components] 'my_comp': No component registered for that name"): + await page.evaluate(test_js) + + await page.close() diff --git a/tox.ini b/tox.ini index 7bad46dd..8e80ff18 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,10 @@ deps = django50: Django>=5.0,<5.1 pytest pytest-xdist + playwright + requests + types-requests + whitenoise commands = pytest {posargs} [testenv:flake8] @@ -42,13 +46,20 @@ deps = isort commands = isort --check-only --diff src/django_components [testenv:coverage] -deps = pytest-coverage +deps = + pytest-coverage + playwright + requests + types-requests + whitenoise commands = coverage run --branch -m pytest coverage report -m --fail-under=97 [testenv:mypy] -deps = mypy +deps = + mypy + types-requests commands = mypy . [testenv:black]