From 3600c8b28d6afcffb19a485be932da33d812ab10 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Thu, 4 Apr 2024 11:27:33 +0200 Subject: [PATCH 01/30] bla --- docs/media_files_handling.md | 73 ++++++++++++++++++++++++++++++++++++ server/lorenz.env | 6 +-- 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 docs/media_files_handling.md diff --git a/docs/media_files_handling.md b/docs/media_files_handling.md new file mode 100644 index 00000000..40551bc9 --- /dev/null +++ b/docs/media_files_handling.md @@ -0,0 +1,73 @@ +# Files handling + +This document describes how files are handled in this appication. + +# Types of files + +static files: files that are not changed by the application, e.g. images, fonts, etc.¨ + +### content documents: + +Files that belong to the content and are managed by the content editors in the CMS (pdf, excel, word, etc.) + +### user documents: + +Files that are uploaded by the users (pdf, etc.). Therefore not visible in the CMS. +Images are handled seprately from documents since images require additional processing (resizing, cropping, etc.). +Visible in the django admin. + +### content images: + +Images that belong to the content and are managed by the content editors in the CMS. + +### user images: + +Images that are uploaded by the users. Therefore not visible in the CMS. Visible in the django admin. + +## Static files + +These files are publicly served on S3. + +## Content documents + +These files are part of the content. Such as a pdf thas cointains additional information to a course. +These files are not publicly available. The content files are uploaded by the editors in the wagtail cms. + +https://www.hacksoft.io/blog/direct-to-s3-file-upload-with-django + +Django handles the permissions to these files. Via a view django checks if the user has permissions to access the file, +and gerates a temporary url that is valid for a limited time. Still the documents are served by django. This done for +usability reasons. The user sees the url mydomain.com/media/documents/ and not a url to S3. Therefore the +user can share the url with other users. (still they need to login and have the permissions to access the file) + +The downside of this is that the django server processes these files. (could be circumvented by django-sendfile). + +![](./assets/files-presign.png) + +- These Files are handled stored as wagtail documents. As a model and the file itself is stored in S3. + +### Frontend access to content documents + +For the frontend django generates a fixed url per file /media/documents/ + +When the frontend requests this file, django checks if the user has permissions to access the file. +If so, django generates a temporary url that is valid for a limited time. Then sends a redirect to the frontend. + +In this waz the frontend does not need to know about the permissions. Content grapql can be cached if needed and urls +can be shared by the users. + +content_documents +user_documents + +public files + +## User documents + +- User uploaded files are stored in S3. but the permissions is handled by django. Same process as content files. + +Same process as content files. But the url is /media/user-uploads/ +And the files are not managed by Wagtail. Due to another model, they are not visible to the user in the CMS. + +## Content images + +Content Images are served directly from S3. The permissions are handled by dja diff --git a/server/lorenz.env b/server/lorenz.env index dca09e30..3a3bc88a 100644 --- a/server/lorenz.env +++ b/server/lorenz.env @@ -4,10 +4,10 @@ export DATABASE_URL=postgres://skillbox:skillbox@localhost:5432/skillbox export DEBUG=True export ENABLE_SILKY=False export SECRET_KEY=FOOBAR -export USE_AWS=False +export USE_AWS=True export WAGTAILADMIN_BASE_URL=/ export ALLOW_BETA_LOGIN=True -export USE_404_FALLBACK_IMAGE=True - +export AWS_STORAGE_BUCKET_NAME=skillbox-files-preprod +export AWS_REGION=eu-central-1 #export THEME=my-kv #export APP_FLAVOR=my-kv From 080d9f92d0b10e0aa333cf1633eb72079e495b5a Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Mon, 8 Apr 2024 18:36:20 +0200 Subject: [PATCH 02/30] Add wagtail image component. And modify resolvers --- client/src/components/modules/Module.vue | 10 ++-- .../src/components/modules/ModuleTeaser.vue | 10 ++-- client/src/components/ui/WagtailImage.vue | 56 +++++++++++++++++++ server/books/schema/interfaces/module.py | 6 +- server/core/urls.py | 10 +++- 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 client/src/components/ui/WagtailImage.vue diff --git a/client/src/components/modules/Module.vue b/client/src/components/modules/Module.vue index 4222df2d..83cb0008 100644 --- a/client/src/components/modules/Module.vue +++ b/client/src/components/modules/Module.vue @@ -28,11 +28,10 @@ {{ module.title }}
- + + + +
-
+ + + + +

{{ metaTitle }}

@@ -33,6 +34,7 @@ import Pill from '@/components/ui/Pill.vue'; import { ModuleCategoryNode, ModuleLevelNode } from '@/__generated__/graphql'; import { computed } from '@vue/reactivity'; +import WagtailImage from "@/components/ui/WagtailImage.vue"; export interface Props { metaTitle: string; diff --git a/client/src/components/ui/WagtailImage.vue b/client/src/components/ui/WagtailImage.vue new file mode 100644 index 00000000..7b83105b --- /dev/null +++ b/client/src/components/ui/WagtailImage.vue @@ -0,0 +1,56 @@ + + + diff --git a/server/books/schema/interfaces/module.py b/server/books/schema/interfaces/module.py index 87d7919e..b323c39d 100644 --- a/server/books/schema/interfaces/module.py +++ b/server/books/schema/interfaces/module.py @@ -1,6 +1,7 @@ import graphene from graphene import relay - +from wagtail.images.models import Image +from api.graphene_wagtail import generate_image_url class ModuleInterface(relay.Node): pk = graphene.Int() @@ -14,4 +15,5 @@ class ModuleInterface(relay.Node): @staticmethod def resolve_hero_image(parent, info, **kwargs): if parent.hero_image: - return parent.hero_image.file.url + image = Image.objects.get(id=parent.hero_image.id) + return generate_image_url(image, 'original') diff --git a/server/core/urls.py b/server/core/urls.py index c062f80b..25bce87f 100644 --- a/server/core/urls.py +++ b/server/core/urls.py @@ -6,23 +6,31 @@ from django.views.generic import RedirectView from wagtail.admin import urls as wagtailadmin_urls from wagtail import urls as wagtail_urls from wagtail.documents import urls as wagtaildocs_urls +#from wagtail.images import urls as wagtailimages_urls +from wagtail.images.views.serve import ServeView from wagtailautocomplete.urls.admin import urlpatterns as autocomplete_admin_urls from core import views -from core.views import override_wagtailadmin_explore_default_ordering +from core.views import override_wagtailadmin_explore_default_ordering, user_image urlpatterns = [ # django admin re_path(r"^guru/", admin.site.urls), re_path(r"^statistics/", include("statistics.urls", namespace="statistics")), # wagtail + re_path(r'^api/images/([^/]*)/(\d*)/([^/]*)/[^/]*$', ServeView.as_view(action='redirect'), name='wagtailimages_serve'), + re_path(r"^cms/autocomplete/", include(autocomplete_admin_urls)), re_path(r"^cms/pages/(\d+)/$", override_wagtailadmin_explore_default_ordering), re_path(r"^cms/", include(wagtailadmin_urls)), re_path(r"^documents/", include(wagtaildocs_urls)), + + #re_path(r"^images/", include(wagtailimages_urls)), + # graphql backend re_path(r"^api/", include("api.urls", namespace="api")), + # favicon re_path( r"^favicon\.ico$", From 9645aed5f2e2723d7c16f2a79677ffc18f73e1a3 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Tue, 9 Apr 2024 14:18:46 +0200 Subject: [PATCH 03/30] Refactor to vue3 --- client/src/components/ui/WagtailImage.vue | 103 +++++++++++++--------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/client/src/components/ui/WagtailImage.vue b/client/src/components/ui/WagtailImage.vue index 7b83105b..f1918faa 100644 --- a/client/src/components/ui/WagtailImage.vue +++ b/client/src/components/ui/WagtailImage.vue @@ -1,56 +1,71 @@ - From 6d920c2358b49d51a3b5eff8e6d1b84184449c07 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Tue, 9 Apr 2024 14:20:38 +0200 Subject: [PATCH 04/30] Use WagtailImage in Image content block --- client/src/components/content-blocks/ImageBlock.vue | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/src/components/content-blocks/ImageBlock.vue b/client/src/components/content-blocks/ImageBlock.vue index db50038e..5543f3c2 100644 --- a/client/src/components/content-blocks/ImageBlock.vue +++ b/client/src/components/content-blocks/ImageBlock.vue @@ -1,15 +1,18 @@ + + diff --git a/server/core/settings.py b/server/core/settings.py index a610c84b..0d8d31f2 100644 --- a/server/core/settings.py +++ b/server/core/settings.py @@ -266,7 +266,6 @@ AWS_S3_CUSTOM_DOMAIN = "{}.s3-{}.amazonaws.com".format( ) if USE_AWS: STORAGE_BACKEND = "storages.backends.s3boto3.S3Boto3Storage" - # use with cloudfront MEDIA_URL = "https://%s/" % AWS_S3_CUSTOM_DOMAIN else: STORAGE_BACKEND = "django.core.files.storage.FileSystemStorage" @@ -388,6 +387,7 @@ WAGTAILSEARCH_BACKENDS = { } } WAGTAILDOCS_DOCUMENT_MODEL = "books.CustomDocument" +WAGTAILADMIN_BASE_URL = "/cms/" GRAPHQL_QUERIES_DIR = os.path.join( From 3cef9d10c9d84f0fccfa9fa6b0b2d0a579a58d10 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Tue, 23 Apr 2024 16:48:41 +0200 Subject: [PATCH 07/30] Add custom serve view. --- client/src/components/ui/WagtailImage.vue | 12 +++-- local-setup-for-cypress-tests.sh | 33 ++++++++++++++ server/api/graphene_wagtail.py | 5 +-- server/core/urls.py | 6 +-- server/core/views.py | 54 ++++++++++++++++++++++- server/core/wagtail_image.py | 0 6 files changed, 99 insertions(+), 11 deletions(-) create mode 100755 local-setup-for-cypress-tests.sh create mode 100644 server/core/wagtail_image.py diff --git a/client/src/components/ui/WagtailImage.vue b/client/src/components/ui/WagtailImage.vue index 1ad4b225..bade9c11 100644 --- a/client/src/components/ui/WagtailImage.vue +++ b/client/src/components/ui/WagtailImage.vue @@ -4,6 +4,7 @@ :src="modifiedUrl" :alt="alt" class="wagtail-image__image" + loading="eager" v-show="loaded" ref="imgElement" @load="loaded = true" @@ -85,15 +86,20 @@ onMounted(updateDimensions); @import 'styles/helpers'; .wagtail-image { - max-width: 100%; + overflow: hidden; + &__image { - max-width: 100%; + width: 100%; + height: auto; /* Keep the image's aspect ratio intact */ + min-height: 100%; + } &__placeholder { background-color: $color-silver-light; - max-width: 100%; + width: 100%; + height: 100%; } } diff --git a/local-setup-for-cypress-tests.sh b/local-setup-for-cypress-tests.sh new file mode 100755 index 00000000..43f4ca1f --- /dev/null +++ b/local-setup-for-cypress-tests.sh @@ -0,0 +1,33 @@ +#!/bin/bash +#!/bin/bash +export SECRET_KEY=abcd1234 +export DATABASE_HOST=localhost +export DATABASE_USER=postgres +export PGPASSWORD=postgres +export DATABASE_NAME=skillbox_test_cypress +export DATABASE_PORT=5432 +export DATABASE_URL=postgres://$DATABASE_USER:$PGPASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME +export DEBUG=True +export USE_AWS=False +export SERVE_VIA_WEBPACK=False +export OAUTH_CLIENT_ID=1111111-222222-333-3444444 +export OAUTH_CLIENT_SECRET=Abcd1234! +export OAUTH_ACCESS_TOKEN_URL=https://hepverlag-cms.grape.novu.ch/oauth/token +export OAUTH_AUTHORIZE_URL=https://hepverlag-cms.grape.novu.ch/oauth/authorize +export OAUTH_API_BASE_URL=https://hepverlag-cms.grape.novu.ch/ +export OAUTH_LOCAL_REDIRECT_URI=http://localhost:8000/api/oauth/callback/ +export NODE_OPTIONS=--max_old_space_size=3072 + +export DATABASE_HOST=localhost +export DATABASE_PORT=5432 +export DATABASE_URL=postgres://$DATABASE_USER:$PG_PASSWORD@$DATABASE_HOST:$DATABASE_PORT/$DATABASE_NAME +psql -U $DATABASE_USER -h $DATABASE_HOST -c "drop database $DATABASE_NAME" + +#npm install --prefix client +#npm run "install:cypress" --prefix client +psql -U $DATABASE_USER -h $DATABASE_HOST -c "create database $DATABASE_NAME" +python server/manage.py dummy_data +python server/manage.py runserver & +npm run dev --prefix client & +cd client +/node_modules/.bin/cypress run diff --git a/server/api/graphene_wagtail.py b/server/api/graphene_wagtail.py index da60dfce..3f00156d 100644 --- a/server/api/graphene_wagtail.py +++ b/server/api/graphene_wagtail.py @@ -12,6 +12,7 @@ from wagtail.images.models import Image from assignments.models import Assignment from basicknowledge.models import BasicKnowledge from books.models import CustomDocument +from core.wagtail_image import get_image_json from surveys.models import Survey logger = logging.getLogger(__name__) @@ -49,9 +50,7 @@ def augment_fields(raw_data): if _type == 'image_block': _value = data['value'] image = Image.objects.get(id=_value) - value = { - 'path': generate_image_url(image, 'original'), - } + value = get_image_json(image.id) data['value'] = value if _type == 'assignment': _value = data['value'] diff --git a/server/core/urls.py b/server/core/urls.py index 25bce87f..c4a8ab2d 100644 --- a/server/core/urls.py +++ b/server/core/urls.py @@ -7,19 +7,19 @@ from wagtail.admin import urls as wagtailadmin_urls from wagtail import urls as wagtail_urls from wagtail.documents import urls as wagtaildocs_urls #from wagtail.images import urls as wagtailimages_urls -from wagtail.images.views.serve import ServeView +from core.views import CustomImageServeView from wagtailautocomplete.urls.admin import urlpatterns as autocomplete_admin_urls from core import views -from core.views import override_wagtailadmin_explore_default_ordering, user_image +from core.views import override_wagtailadmin_explore_default_ordering urlpatterns = [ # django admin re_path(r"^guru/", admin.site.urls), re_path(r"^statistics/", include("statistics.urls", namespace="statistics")), # wagtail - re_path(r'^api/images/([^/]*)/(\d*)/([^/]*)/[^/]*$', ServeView.as_view(action='redirect'), name='wagtailimages_serve'), + re_path(r'^api/images/([^/]*)/(\d*)/([^/]*)/[^/]*$', CustomImageServeView.as_view(action='redirect'), name='wagtailimages_serve'), re_path(r"^cms/autocomplete/", include(autocomplete_admin_urls)), re_path(r"^cms/pages/(\d+)/$", override_wagtailadmin_explore_default_ordering), diff --git a/server/core/views.py b/server/core/views.py index e808a13f..e489c49c 100644 --- a/server/core/views.py +++ b/server/core/views.py @@ -1,15 +1,31 @@ +import imghdr +from wsgiref.util import FileWrapper + import requests -from core.logger import get_logger from django.conf import settings +from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin +from django.core.exceptions import PermissionDenied +from django.http import HttpResponse, StreamingHttpResponse from django.http.request import HttpRequest -from django.http.response import HttpResponse, HttpResponseRedirect +from django.http.response import HttpResponseRedirect +from django.shortcuts import get_object_or_404 from django.shortcuts import render +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_control from django.views.decorators.csrf import ensure_csrf_cookie from graphene_django.views import GraphQLView from graphql import get_operation_ast, parse from sentry_sdk.api import start_transaction from wagtail.admin.views.pages import listing +from wagtail.images import get_image_model +from wagtail.images.exceptions import InvalidFilterSpecError +from wagtail.images.models import Image +from wagtail.images.models import SourceImageIOError +from wagtail.images.utils import verify_signature +from wagtail.images.views.serve import ServeView + +from core.logger import get_logger logger = get_logger(__name__) @@ -68,3 +84,37 @@ def override_wagtailadmin_explore_default_ordering(request, parent_page_id): return HttpResponseRedirect(request.path_info + "?ordering=ord") return listing.IndexView.as_view()(request, parent_page_id) + +class CustomImageServeView(ServeView): + ''' Taken from wagtail.images.views.serve.ServeView the only change is that we use original for the filter_spec_for_signature''' + model = get_image_model() + action = "serve" + key = None + + @method_decorator(cache_control(max_age=3600, public=True)) + def get(self, request, signature, image_id, filter_spec, filename=None): + # This variable is the only change to the wagtail implementation + filter_spec_for_signature = 'original' + + if not verify_signature( + signature.encode(), image_id, filter_spec_for_signature, key=self.key + ): + raise PermissionDenied + + image = get_object_or_404(self.model, id=image_id) + + # Get/generate the rendition + try: + rendition = image.get_rendition(filter_spec) + except SourceImageIOError: + return HttpResponse( + "Source image file not found", content_type="text/plain", status=410 + ) + except InvalidFilterSpecError: + return HttpResponse( + "Invalid filter spec: " + filter_spec, + content_type="text/plain", + status=400, + ) + + return getattr(self, self.action)(rendition) diff --git a/server/core/wagtail_image.py b/server/core/wagtail_image.py new file mode 100644 index 00000000..e69de29b From f4700635e3b4480aae02de5f900e87973c23c96c Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Tue, 23 Apr 2024 17:39:41 +0200 Subject: [PATCH 08/30] Add placeholder aspect ratio --- .../components/content-blocks/ImageBlock.vue | 6 ++- client/src/components/ui/WagtailImage.vue | 41 +++++++++++++++---- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/client/src/components/content-blocks/ImageBlock.vue b/client/src/components/content-blocks/ImageBlock.vue index 5543f3c2..351b381f 100644 --- a/client/src/components/content-blocks/ImageBlock.vue +++ b/client/src/components/content-blocks/ImageBlock.vue @@ -1,7 +1,9 @@ @@ -22,6 +24,8 @@ import { ref, computed, onMounted } from 'vue'; const props = defineProps({ src: String, alt: String(''), + originalWidth: Number, + originalHeight: Number, }); const imgElement = ref(null); @@ -30,7 +34,6 @@ const height = ref(0); const loaded = ref(false); const modifiedUrl = computed(() => { - console.log('x, y, hdpi, retina:', width.value, height.value, isHighDensity(), isRetina()); const density = isHighDensity() ? 2 : 1; if (width.value) { @@ -52,6 +55,32 @@ const updateDimensions = () => { } }; +const handleLoad = () => { + loaded.value = true; // Set loaded to true when the image loads +}; + +const placeholderStyle = computed(() => { + const styles = { + width: '100%', + height: '100%', + }; + if (width.value) { + const scalingFactor = width.value / props.originalWidth; + console.log(props.originalWidth, width.value); + const scaledHeight = Math.round(props.originalHeight * scalingFactor); + const scaledWidth = Math.round(props.originalWidth * scalingFactor); + + if (props.originalWidth) { + styles.width = `${scaledWidth}px`; + } + if (props.originalHeight) { + styles.height = `${scaledHeight}px`; + } + console.log(styles); + return styles; + } +}); + const isHighDensity = () => { return ( (window.matchMedia && @@ -88,12 +117,10 @@ onMounted(updateDimensions); .wagtail-image { overflow: hidden; - &__image { width: 100%; - height: auto; /* Keep the image's aspect ratio intact */ - min-height: 100%; - + //height: 100%; + height: auto; /* Keep the image's aspect ratio intact */ } &__placeholder { From e9c172e265fab167ba43296f71852f87260628a3 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Thu, 25 Apr 2024 10:52:23 +0200 Subject: [PATCH 09/30] Fix readme how to generate Graphql schema --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 406e4ab0..59dcedc4 100644 --- a/README.md +++ b/README.md @@ -244,10 +244,24 @@ python manage.py export_schema_for_cypress ### Generate GraphQL SDL Document + +for linux: +```bash +cd server +./graphql-schema.sh && npm run codegen --../prefix client ``` -python manage.py export_schema_graphql + +For macOS: (there is a problem with the sed command) + +```bash +cd server +./macos-graphql-schema.sh && npm run codegen --../prefix client + ``` + + + ## Backup to S3 From https://pawelurbanek.com/heroku-postgresql-s3-backup From cd0eb081538902147c8064c995942b9fc8902fc5 Mon Sep 17 00:00:00 2001 From: Lorenz Padberg Date: Thu, 25 Apr 2024 11:24:40 +0200 Subject: [PATCH 10/30] Use WagtailImageNode for Module hero Image --- .../components/content-blocks/ImageBlock.vue | 2 +- client/src/components/modules/Module.vue | 8 +++-- client/src/components/ui/WagtailImage.vue | 13 --------- .../src/graphql/gql/fragments/moduleParts.gql | 7 +++++ server/api/graphene_wagtail.py | 24 +++++++++++++-- server/books/schema/interfaces/module.py | 10 +++++++ server/books/schema/nodes/__init__.py | 1 + server/books/schema/nodes/module.py | 6 ++++ server/books/schema/nodes/wagtail_image.py | 29 +++++++++++++++++++ server/books/tests/queries.py | 6 ++++ server/core/wagtail_image.py | 0 11 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 server/books/schema/nodes/wagtail_image.py delete mode 100644 server/core/wagtail_image.py diff --git a/client/src/components/content-blocks/ImageBlock.vue b/client/src/components/content-blocks/ImageBlock.vue index 351b381f..59683de4 100644 --- a/client/src/components/content-blocks/ImageBlock.vue +++ b/client/src/components/content-blocks/ImageBlock.vue @@ -1,6 +1,6 @@