From 48642eccc4a21c680624351a525972e1a663ccff Mon Sep 17 00:00:00 2001 From: Matt Westcott Date: Mon, 27 Jun 2022 23:59:24 +0100 Subject: [PATCH] Initial commit --- .circleci/config.yml | 85 +++++++++ .circleci/report_nightly_build_failure.py | 18 ++ .circleci/trigger-nightly-build.sh | 8 + .coveragerc | 24 +++ .github/workflows/test.yml | 66 +++++++ .gitignore | 13 ++ .prettierrc.toml | 2 + CHANGELOG.md | 42 +++++ CONTRIBUTING.md | 1 + LICENSE | 33 ++++ MANIFEST.in | 4 + README.md | 60 +++++++ SECURITY.md | 5 + setup.py | 53 ++++++ testmanage.py | 66 +++++++ tox.ini | 44 +++++ wagtail_freezer/__init__.py | 5 + wagtail_freezer/apps.py | 7 + wagtail_freezer/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/buildstaticsite.py | 100 +++++++++++ wagtail_freezer/models.py | 0 wagtail_freezer/test/__init__.py | 1 + wagtail_freezer/test/apps.py | 7 + wagtail_freezer/test/settings.py | 161 ++++++++++++++++++ wagtail_freezer/test/tests/__init__.py | 0 wagtail_freezer/test/urls.py | 13 ++ 27 files changed, 818 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .circleci/report_nightly_build_failure.py create mode 100755 .circleci/trigger-nightly-build.sh create mode 100644 .coveragerc create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .prettierrc.toml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 setup.py create mode 100644 testmanage.py create mode 100644 tox.ini create mode 100644 wagtail_freezer/__init__.py create mode 100644 wagtail_freezer/apps.py create mode 100644 wagtail_freezer/management/__init__.py create mode 100644 wagtail_freezer/management/commands/__init__.py create mode 100644 wagtail_freezer/management/commands/buildstaticsite.py create mode 100644 wagtail_freezer/models.py create mode 100644 wagtail_freezer/test/__init__.py create mode 100644 wagtail_freezer/test/apps.py create mode 100644 wagtail_freezer/test/settings.py create mode 100644 wagtail_freezer/test/tests/__init__.py create mode 100644 wagtail_freezer/test/urls.py diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..acff130 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,85 @@ +version: 2 + +jobs: + flake8: + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: pip install flake8 + - run: flake8 wagtail_freezer + + prettier: + docker: + - image: circleci/node:16 + steps: + - checkout: + + - type: cache-restore + keys: + - node-modules-{{ .Branch }}-{{ checksum "package-lock.json" }} + - node-modules-{{ .Branch }}- + - node-modules-master- + + - run: npm install + - run: npm run lint + + - type: cache-save + key: node-modules-{{ .Branch }}-{{ checksum "package-lock.json" }} + paths: + - "node_modules" + + test: + docker: + - image: circleci/python:3.8 + steps: + - checkout + + - type: cache-restore + keys: + - pip-{{ .Branch }}- + - pip-master- + + - run: pip install -e .[testing] + + - type: cache-save + key: pip-{{ .Branch }}-{{ epoch }} + paths: + - "~/.cache/pip" + + - run: python testmanage.py test + + nightly-wagtail-test: + docker: + - image: circleci/python:3.8 + steps: + - checkout + - run: git clone git@github.com:wagtail/wagtail.git + + - run: pip install -e .[testing] + - run: pip install ./wagtail + + - run: python testmanage.py test + + - run: + when: on_fail + command: python ./.circleci/report_nightly_build_failure.py + +workflows: + version: 2 + test: + jobs: + - flake8 + - prettier + - test + + nightly: + jobs: + - nightly-wagtail-test + triggers: + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - master diff --git a/.circleci/report_nightly_build_failure.py b/.circleci/report_nightly_build_failure.py new file mode 100644 index 0000000..ef54fe5 --- /dev/null +++ b/.circleci/report_nightly_build_failure.py @@ -0,0 +1,18 @@ +""" +Called by CircleCI when the nightly build fails. + +This reports an error to the #nightly-build-failures Slack channel. +""" +import os +import requests + +if 'SLACK_WEBHOOK_URL' in os.environ: + print("Reporting to #nightly-build-failures slack channel") + response = requests.post(os.environ['SLACK_WEBHOOK_URL'], json={ + "text": "A Nightly build failed. See " + os.environ['CIRCLE_BUILD_URL'], + }) + + print("Slack responded with:", response) + +else: + print("Unable to report to #nightly-build-failures slack channel because SLACK_WEBHOOK_URL is not set") diff --git a/.circleci/trigger-nightly-build.sh b/.circleci/trigger-nightly-build.sh new file mode 100755 index 0000000..eb21c16 --- /dev/null +++ b/.circleci/trigger-nightly-build.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Triggers a test run against the master version of Wagtail + +# This job will is scheduled in the config.yml, this script is here to help test the job + +curl -u ${CIRCLE_API_USER_TOKEN}: \ + -d build_parameters[CIRCLE_JOB]=nightly-wagtail-test \ + https://circleci.com/api/v1.1/project/github/gasman/wagtail-freezer/tree/master diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..b3ef9ee --- /dev/null +++ b/.coveragerc @@ -0,0 +1,24 @@ +[run] +branch = True +include = wagtail_freezer/* +omit = */migrations/*,*/tests/* + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3dceb60 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,66 @@ +name: Freezer CI + +on: + push: + branches: + - main + - master + - 'stable/**' + + pull_request: + +jobs: + test-sqlite: + runs-on: ubuntu-latest + strategy: + matrix: + python: ['3.7', '3.8', '3.9'] + django: ['3.0', '3.1', '3.2'] + wagtail: ['2.14', '2.15'] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install Tox + run: | + python -m pip install tox + - name: Test + run: | + tox + env: + TOXENV: python${{ matrix.python }}-django${{ matrix.django }}-wagtail${{ matrix.wagtail }}-sqlite + + test-postgres: + runs-on: ubuntu-latest + strategy: + matrix: + python: ['3.7', '3.8', '3.9'] + django: ['3.0', '3.1', '3.2'] + wagtail: ['2.14', '2.15'] + postgres: ['10.8'] + + services: + postgres: + image: postgres:${{ matrix.postgres }} + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Install Tox + run: | + python -m pip install tox + - name: Test + run: | + tox + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/wagtail_freezer + TOXENV: python${{ matrix.python }}-django${{ matrix.django }}-wagtail${{ matrix.wagtail }}-postgres diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5a0efd1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.pyc +/build +/dist +/wagtail_freezer.egg-info +/.coverage +/htmlcov +/.tox +/venv +/.vscode +/site +/test_wagtail_freezer.db +/test-static +/test-media diff --git a/.prettierrc.toml b/.prettierrc.toml new file mode 100644 index 0000000..009474b --- /dev/null +++ b/.prettierrc.toml @@ -0,0 +1,2 @@ +tabWidth = 4 +singleQuote = true diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8b93be4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,42 @@ +# wagtail-freezer Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2022-06-27 + +### Added + +- ... + +### Changed + +- ... + +### Removed + +- ... + + + \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e47a5bd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +# Contributing to wagtail-freezer diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2afe32b --- /dev/null +++ b/LICENSE @@ -0,0 +1,33 @@ + + + +BSD License + +Copyright (c) 2022, Matt Westcott +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ff1288f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE *.rst *.txt *.md +graft wagtail_freezer +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/README.md b/README.md new file mode 100644 index 0000000..f7b1b7b --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# Wagtail Freezer + +Generates static HTML sites from a Wagtail project + + +[![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) + +[![PyPI version](https://badge.fury.io/py/wagtail-freezer.svg)](https://badge.fury.io/py/wagtail-freezer) +[![wagtail-freezer CI](https://github.com/gasman/wagtail-freezer/actions/workflows/test.yml/badge.svg)](https://github.com/gasman/wagtail-freezer/actions/workflows/test.yml) + +## Links + +- [Documentation](https://github.com/gasman/wagtail-freezer/blob/main/README.md) +- [Changelog](https://github.com/gasman/wagtail-freezer/blob/main/CHANGELOG.md) +- [Contributing](https://github.com/gasman/wagtail-freezer/blob/main/CHANGELOG.md) +- [Discussions](https://github.com/gasman/wagtail-freezer/discussions) +- [Security](https://github.com/gasman/wagtail-freezer/security) + +## Supported versions + +- Python 3.7 - 3.10 +- Django 3.x +- Wagtail 3.x + +## Installation + +- `pip install wagtail-freezer` +- Add `"wagtail_freezer"` to INSTALLED_APPS +- Add a `FREEZER_BUILD_DIR` setting indicating where the static files will be output. To write into a folder named `build` at the project root, use: + + FREEZER_BUILD_DIR = os.path.join(BASE_DIR, "build") + +## Usage + +Run `./manage.py buildstaticsite`. This will generate one folder per site within FREEZER_BUILD_DIR, with subfolders making up the page tree and the pages themselves saved as `index.html` at the appropriate point. + +While building the static files, wagtail-freezer will scan the HTML for any `href` or `src` attributes that reference files under `STATIC_URL` or `MEDIA_URL`, and copy these files to corresponding folders under the site root. This step only takes place if `STATIC_URL` or `MEDIA_URL` are local URLs beginning with '/'. + +If you have additional static / media files that can't be found by parsing HTML (for example, images referenced within CSS, JavaScript or JSON), you can provide a `freezer_follow_urls` method on the page model that returns a list of media / static URLs to be followed: + +```python +class HomePage(Page): + @property + def freezer_follow_urls(self): + urls = ['/static/images/background.jpg'] + for item in self.playlist.select_related('video'): + urls.append(item.video.url) + return urls +``` + +## Deploying + +When you're happy with how the local static site works (test it by running `python -m http.server` from the root folder), you can deploy it to Amazon S3 by installing the AWS command line tool (`pip install awscli`), creating a bucket [configured for static website hosting](https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteHosting.html), and running: + + aws s3 sync build/localhost s3://mysite.example.com/ --acl public-read + + +## Limitations + +wagtail-freezer was created as a "minimum viable product" substitute for static site generators such as [django-bakery](https://django-bakery.readthedocs.io/), which at the time of writing are lagging behind in support for current Django (and Wagtail) versions. It has only been tested against very simple sites, and will probably not work with custom URL routes (RoutablePageMixin), pages with multiple preview modes (wagtail.contrib.forms, although that's not too usable on a static site anyhow), non-standard middlewares and no doubt lots of other things. Use at your own risk! diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c696f97 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,5 @@ +# Security + +We take the security of Wagtail, and related packages we maintain, seriously. If you have found a security issue with any of our projects please email us at security@wagtail.org so we can work together to find and patch the issue. We appreciate responsible disclosure with any security related issues, so please contact us first before creating a Github issue. + +If you want to send an encrypted email (optional), the public key ID for security@wagtail.org is 0xbed227b4daf93ff9, and this public key is available from most commonly-used keyservers. diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..93218f0 --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +from os import path + +from setuptools import find_packages, setup + +from wagtail_freezer import __version__ + + +this_directory = path.abspath(path.dirname(__file__)) +with open(path.join(this_directory, "README.md"), encoding="utf-8") as f: + long_description = f.read() + +setup( + name="wagtail-freezer", + version=__version__, + description="Generates static HTML sites from a Wagtail project", + long_description=long_description, + long_description_content_type='text/markdown', + author="Matt Westcott", + author_email="matthew@torchbox.com", + url="", + packages=find_packages(), + include_package_data=True, + license="BSD", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Framework :: Django", + "Framework :: Django :: 3.0", + "Framework :: Django :: 3.1", + "Framework :: Django :: 3.2", + "Framework :: Wagtail", + "Framework :: Wagtail :: 3", + ], + install_requires=[ + "Django>=3.0,<4.0", + "Wagtail>=3.0,<4.0", + "beautifulsoup4>=4.9,<5.0", + ], + extras_require={ + "testing": ["dj-database-url==0.5.0"], + }, + zip_safe=False, +) diff --git a/testmanage.py b/testmanage.py new file mode 100644 index 0000000..56663d5 --- /dev/null +++ b/testmanage.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +import argparse +import os +import shutil +import sys +import warnings + +from django.core.management import execute_from_command_line + + +os.environ["DJANGO_SETTINGS_MODULE"] = "wagtail_freezer.test.settings" + + +def make_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--deprecation", + choices=["all", "pending", "imminent", "none"], + default="imminent", + ) + return parser + + +def parse_args(args=None): + return make_parser().parse_known_args(args) + + +def runtests(): + args, rest = parse_args() + + only_wagtail = r"^wagtail(\.|$)" + if args.deprecation == "all": + # Show all deprecation warnings from all packages + warnings.simplefilter("default", DeprecationWarning) + warnings.simplefilter("default", PendingDeprecationWarning) + elif args.deprecation == "pending": + # Show all deprecation warnings from wagtail + warnings.filterwarnings( + "default", category=DeprecationWarning, module=only_wagtail + ) + warnings.filterwarnings( + "default", category=PendingDeprecationWarning, module=only_wagtail + ) + elif args.deprecation == "imminent": + # Show only imminent deprecation warnings from wagtail + warnings.filterwarnings( + "default", category=DeprecationWarning, module=only_wagtail + ) + elif args.deprecation == "none": + # Deprecation warnings are ignored by default + pass + + argv = [sys.argv[0]] + rest + + try: + execute_from_command_line(argv) + finally: + from wagtail.tests.settings import STATIC_ROOT, MEDIA_ROOT + + shutil.rmtree(STATIC_ROOT, ignore_errors=True) + shutil.rmtree(MEDIA_ROOT, ignore_errors=True) + + +if __name__ == "__main__": + runtests() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..11dfde6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,44 @@ +[tox] +skipsdist = True +usedevelop = True + +envlist = python{3.7,3.8,3.9}-django{3.0,3.1,3.2,master}-wagtail{2.14,2.15,master}-{sqlite,postgres} + +[flake8] +# E501: Line too long +# W503: line break before binary operator (superseded by W504 line break after binary operator) +ignore = E501,W503 +exclude = migrations,node_modules + +[testenv] +install_command = pip install -e ".[testing]" -U {opts} {packages} +commands = coverage run testmanage.py test --deprecation all + +basepython = + python3.7: python3.7 + python3.8: python3.8 + python3.9: python3.9 + +deps = + coverage + + django3.0: Django>=3.0,<3.1 + django3.1: Django>=3.1,<3.2 + django3.2: Django>=3.2,<4.0 + + djangomaster: git+https://github.com/django/django.git@master#egg=Django + djangomaster: git+https://github.com/wagtail/django-modelcluster.git + + wagtail2.14: wagtail>=2.14,<2.15 + wagtail2.15: wagtail==2.15rc1 + wagtailmaster: git+https://github.com/wagtail/wagtail.git + + postgres: psycopg2>=2.6 + +setenv = + postgres: DATABASE_URL={env:DATABASE_URL:postgres:///wagtail_freezer} + +[testenv:flake8] +basepython=python3.7 +deps=flake8>=2.2.0 +commands=flake8 wagtail_freezer diff --git a/wagtail_freezer/__init__.py b/wagtail_freezer/__init__.py new file mode 100644 index 0000000..e94919e --- /dev/null +++ b/wagtail_freezer/__init__.py @@ -0,0 +1,5 @@ +default_app_config = "wagtail_freezer.apps.WagtailFreezerAppConfig" + + +VERSION = (0, 1, 0) +__version__ = ".".join(map(str, VERSION)) diff --git a/wagtail_freezer/apps.py b/wagtail_freezer/apps.py new file mode 100644 index 0000000..1914d1a --- /dev/null +++ b/wagtail_freezer/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class WagtailFreezerAppConfig(AppConfig): + label = "wagtail_freezer" + name = "wagtail_freezer" + verbose_name = "Wagtail Freezer" diff --git a/wagtail_freezer/management/__init__.py b/wagtail_freezer/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_freezer/management/commands/__init__.py b/wagtail_freezer/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_freezer/management/commands/buildstaticsite.py b/wagtail_freezer/management/commands/buildstaticsite.py new file mode 100644 index 0000000..ec579cd --- /dev/null +++ b/wagtail_freezer/management/commands/buildstaticsite.py @@ -0,0 +1,100 @@ +from pathlib import Path +from shutil import copyfile, rmtree + +from bs4 import BeautifulSoup +from django.core.exceptions import ImproperlyConfigured +from django.core.handlers.base import BaseHandler +from django.core.handlers.wsgi import WSGIRequest +from django.core.management.base import BaseCommand +from django.conf import settings +from django.contrib.staticfiles import finders +from wagtail.models import Site + +class Command(BaseCommand): + help = "Generate a static HTML version of this Wagtail site" + + def follow_url(self, url): + if url.startswith(self.static_assets_url): + self.static_assets.add(url[len(self.static_assets_url):]) + elif url.startswith(self.media_url): + self.media.add(url[len(self.media_url):]) + + def handle(self, *args, **options): + try: + static_root = Path(settings.FREEZER_BUILD_DIR) + except AttributeError: + raise ImproperlyConfigured("FREEZER_BUILD_DIR must be defined in settings") + + self.static_assets_url = getattr(settings, "STATIC_URL", "") + self.copy_static_assets = self.static_assets_url.startswith("/") + self.media_url = getattr(settings, "MEDIA_URL", "") + self.copy_media = self.media_url.startswith("/") + + sites = Site.objects.all() + + for site in sites: + self.static_assets = set() + self.media = set() + site_static_root = static_root / site.hostname + rmtree(site_static_root, ignore_errors=True) + + pages = site.root_page.get_descendants(inclusive=True).live().order_by('path').specific() + for page in pages: + relative_path = page.url_path[len(site.root_page.url_path):] + page_path = site_static_root / relative_path + + dummy_meta = page._get_dummy_headers() + request = WSGIRequest(dummy_meta) + + # Add a flag to let middleware know that this is a dummy request. + request.is_dummy = True + + # Build a custom django.core.handlers.BaseHandler subclass that invokes serve() as + # the eventual view function called at the end of the middleware chain, rather than going + # through the URL resolver + class Handler(BaseHandler): + def _get_response(self, request): + response = page.serve(request) + if hasattr(response, "render") and callable(response.render): + response = response.render() + return response + + # Invoke this custom handler. + handler = Handler() + handler.load_middleware() + response = handler.get_response(request) + + page_path.mkdir(parents=True) + with (page_path / "index.html").open(mode='wb') as f: + f.write(response.content) + + if self.copy_static_assets or self.copy_media: + soup = BeautifulSoup(response.content, "html.parser") + for elem in soup.find_all(lambda tag:('href' in tag.attrs or 'src' in tag.attrs)): + for attr in ('href', 'src'): + url = elem.get(attr, "") + if url: + self.follow_url(url) + + if hasattr(page, "freezer_follow_urls"): + for url in page.freezer_follow_urls: + self.follow_url(url) + + if self.static_assets: + destination_base_path = site_static_root / self.static_assets_url[1:] + for asset_path in self.static_assets: + source_file = finders.find(asset_path) + if source_file: + destination_path = destination_base_path / asset_path + destination_path.parent.mkdir(parents=True, exist_ok=True) + copyfile(source_file, destination_path) + + if self.media: + destination_base_path = site_static_root / self.media_url[1:] + media_root = Path(settings.MEDIA_ROOT) + for media_path in self.media: + source_path = media_root / media_path + if source_path.is_file(): + destination_path = destination_base_path / media_path + destination_path.parent.mkdir(parents=True, exist_ok=True) + copyfile(source_path, destination_path) diff --git a/wagtail_freezer/models.py b/wagtail_freezer/models.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_freezer/test/__init__.py b/wagtail_freezer/test/__init__.py new file mode 100644 index 0000000..c60c864 --- /dev/null +++ b/wagtail_freezer/test/__init__.py @@ -0,0 +1 @@ +default_app_config = "wagtail_freezer.test.apps.WagtailFreezerTestAppConfig" diff --git a/wagtail_freezer/test/apps.py b/wagtail_freezer/test/apps.py new file mode 100644 index 0000000..0fbc0f1 --- /dev/null +++ b/wagtail_freezer/test/apps.py @@ -0,0 +1,7 @@ +from django.apps import AppConfig + + +class WagtailFreezerTestAppConfig(AppConfig): + label = "wagtail_freezer_test" + name = "wagtail_freezer.test" + verbose_name = "Wagtail Freezer tests" diff --git a/wagtail_freezer/test/settings.py b/wagtail_freezer/test/settings.py new file mode 100644 index 0000000..dbd8621 --- /dev/null +++ b/wagtail_freezer/test/settings.py @@ -0,0 +1,161 @@ +""" +Django settings for temp project. + +For more information on this file, see +https://docs.djangoproject.com/en/stable/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/stable/ref/settings/ +""" + +import os +import dj_database_url + +# Build paths inside the project like this: os.path.join(PROJECT_DIR, ...) +PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +BASE_DIR = os.path.dirname(PROJECT_DIR) + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/stable/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "c6u0-9c!7nilj_ysatsda0(f@e_2mws2f!6m0n^o*4#*q#kzp)" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ["localhost", "testserver"] + + +# Application definition + +INSTALLED_APPS = [ + "wagtail_freezer", + "wagtail_freezer.test", + "wagtail.contrib.search_promotions", + "wagtail.contrib.forms", + "wagtail.contrib.redirects", + "wagtail.embeds", + "wagtail.users", + "wagtail.snippets", + "wagtail.documents", + "wagtail.images", + "wagtail.search", + "wagtail.admin", + "wagtail.api.v2", + "wagtail.contrib.modeladmin", + "wagtail.contrib.routable_page", + "wagtail.contrib.styleguide", + "wagtail.sites", + "wagtail.core", + "taggit", + "rest_framework", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sitemaps", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "wagtail.contrib.redirects.middleware.RedirectMiddleware", +] + +ROOT_URLCONF = "wagtail_freezer.test.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] + }, + } +] + + +# Using DatabaseCache to make sure that the cache is cleared between tests. +# This prevents false-positives in some wagtail core tests where we are +# changing the 'wagtail_root_paths' key which may cause future tests to fail. +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "cache", + } +} + + +# don't use the intentionally slow default password hasher +PASSWORD_HASHERS = ("django.contrib.auth.hashers.MD5PasswordHasher",) + + +# Database +# https://docs.djangoproject.com/en/stable/ref/settings/#databases + +DATABASES = { + "default": dj_database_url.config(default="sqlite:///test_wagtail_freezer.db"), +} + + +# Password validation +# https://docs.djangoproject.com/en/stable/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/stable/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/stable/howto/static-files/ + +STATICFILES_FINDERS = [ + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", +] + +STATICFILES_DIRS = [os.path.join(PROJECT_DIR, "static")] + +STATIC_ROOT = os.path.join(BASE_DIR, "test-static") +STATIC_URL = "/static/" + +MEDIA_ROOT = os.path.join(BASE_DIR, "test-media") + + +# Wagtail settings + +WAGTAIL_SITE_NAME = "Wagtail Freezer test site" +WAGTAILADMIN_BASE_URL = "http://localhost" diff --git a/wagtail_freezer/test/tests/__init__.py b/wagtail_freezer/test/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wagtail_freezer/test/urls.py b/wagtail_freezer/test/urls.py new file mode 100644 index 0000000..e816acb --- /dev/null +++ b/wagtail_freezer/test/urls.py @@ -0,0 +1,13 @@ +from django.conf.urls import include, url +from django.contrib import admin + +from wagtail.admin import urls as wagtailadmin_urls +from wagtail.documents import urls as wagtaildocs_urls +from wagtail.core import urls as wagtail_urls + +urlpatterns = [ + url(r"^django-admin/", admin.site.urls), + url(r"^admin/", include(wagtailadmin_urls)), + url(r"^documents/", include(wagtaildocs_urls)), + url(r"", include(wagtail_urls)), +]