From 96020bf83dcce7814d5e0e9725ade4bad77baf9b Mon Sep 17 00:00:00 2001 From: Daniel Egger Date: Tue, 5 Jul 2022 13:11:45 +0200 Subject: [PATCH 01/32] performance optimization to get full learningpath content --- scripts/count_queries.py | 34 +++++++++++++++++++ server/vbv_lernwelt/learnpath/models.py | 5 +-- .../learnpath/serializer_helpers.py | 18 +++++++++- server/vbv_lernwelt/learnpath/serializers.py | 5 +-- 4 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 scripts/count_queries.py diff --git a/scripts/count_queries.py b/scripts/count_queries.py new file mode 100644 index 00000000..dc65881c --- /dev/null +++ b/scripts/count_queries.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +import json +import os +import sys + +import django + +sys.path.append("../server") + +os.environ.setdefault("IT_APP_ENVIRONMENT", "development") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.base") +django.setup() + +from vbv_lernwelt.learnpath.models import LearningPath +from vbv_lernwelt.learnpath.serializers import LearningPathSerializer + + +def main(): + from django.conf import settings + settings.DEBUG = True + from django.db import connection + from django.db import reset_queries + reset_queries() + + learning_path = LearningPath.objects.filter(slug='versicherungsvermittlerin', locale__language_code='de-CH').first() + + serializer = LearningPathSerializer(learning_path) + print(serializer.data) + print(len(json.dumps(serializer.data))) + print(len(connection.queries)) + + +if __name__ == '__main__': + main() diff --git a/server/vbv_lernwelt/learnpath/models.py b/server/vbv_lernwelt/learnpath/models.py index 67534a3c..0e3d1c57 100644 --- a/server/vbv_lernwelt/learnpath/models.py +++ b/server/vbv_lernwelt/learnpath/models.py @@ -55,8 +55,9 @@ class Topic(Page): @classmethod def get_serializer_class(cls): - return get_it_serializer_class(cls, - field_names=['id', 'title', 'slug', 'type', 'translation_key', 'is_visible', ]) + return get_it_serializer_class( + cls, field_names=['id', 'title', 'slug', 'type', 'translation_key', 'is_visible', ] + ) class Meta: verbose_name = "Topic" diff --git a/server/vbv_lernwelt/learnpath/serializer_helpers.py b/server/vbv_lernwelt/learnpath/serializer_helpers.py index bfaeb48b..5ebfda78 100644 --- a/server/vbv_lernwelt/learnpath/serializer_helpers.py +++ b/server/vbv_lernwelt/learnpath/serializer_helpers.py @@ -20,5 +20,21 @@ class ItBaseSerializer(wagtail_serializers.BaseSerializer): meta_fields = [] + def __init__(self, *args, **kwargs): + self.descendants = kwargs.pop('descendants', None) + super().__init__(*args, **kwargs) + def get_children(self, obj): - return [c.specific.get_serializer_class()(c.specific).data for c in obj.get_children()] + if self.descendants: + children = _get_children(self.descendants, obj) + return [c.specific.get_serializer_class()(c.specific, descendants=self.descendants).data for c in children] + else: + return [c.specific.get_serializer_class()(c.specific).data for c in obj.get_children().specific()] + + +def _get_descendants(pages, obj): + return [c for c in pages if c.path.startswith(obj.path) and c.depth >= obj.depth] + + +def _get_children(pages, obj): + return [c for c in pages if c.path.startswith(obj.path) and obj.depth + 1 == c.depth] diff --git a/server/vbv_lernwelt/learnpath/serializers.py b/server/vbv_lernwelt/learnpath/serializers.py index 3aa8d494..ae4dadd9 100644 --- a/server/vbv_lernwelt/learnpath/serializers.py +++ b/server/vbv_lernwelt/learnpath/serializers.py @@ -1,7 +1,7 @@ from rest_framework import serializers from vbv_lernwelt.learnpath.models import Circle, LearningPath -from vbv_lernwelt.learnpath.serializer_helpers import get_it_serializer_class +from vbv_lernwelt.learnpath.serializer_helpers import get_it_serializer_class, _get_children class LearningPathSerializer(get_it_serializer_class(LearningPath, [])): @@ -10,7 +10,8 @@ class LearningPathSerializer(get_it_serializer_class(LearningPath, [])): meta_fields = [] def get_children(self, obj): - return [c.specific.get_serializer_class()(c.specific).data for c in obj.get_children()] + descendants = [p for p in obj.get_descendants().specific()] + return [c.specific.get_serializer_class()(c.specific, descendants=descendants).data for c in _get_children(descendants, obj)] def get_meta_label(self, obj): return obj._meta.label From 8edea0b92fdfa014a711c1f142daa66297fc1412 Mon Sep 17 00:00:00 2001 From: Daniel Egger Date: Tue, 5 Jul 2022 15:44:11 +0200 Subject: [PATCH 02/32] Add page cache for learningpath pages --- client/src/stores/learningPath.ts | 2 +- scripts/count_queries.py | 7 +++--- server/config/settings/base.py | 23 ++++++++----------- server/vbv_lernwelt/learnpath/models.py | 6 +++++ .../learnpath/serializer_helpers.py | 9 ++++---- server/vbv_lernwelt/learnpath/serializers.py | 21 ----------------- server/vbv_lernwelt/learnpath/signals.py | 16 +++++++++++++ server/vbv_lernwelt/learnpath/urls.py | 6 ++--- server/vbv_lernwelt/learnpath/views.py | 17 +++++--------- 9 files changed, 48 insertions(+), 59 deletions(-) create mode 100644 server/vbv_lernwelt/learnpath/signals.py diff --git a/client/src/stores/learningPath.ts b/client/src/stores/learningPath.ts index 58e231c6..8422a49d 100644 --- a/client/src/stores/learningPath.ts +++ b/client/src/stores/learningPath.ts @@ -25,7 +25,7 @@ export const useLearningPathStore = defineStore({ return this.learningPath; } try { - const learningPathData = await itGet(`/learnpath/api/learningpath/${slug}/`); + const learningPathData = await itGet(`/learnpath/api/page/${slug}/`); const completionData = await itGet(`/api/completion/learning_path/${learningPathData.translation_key}/`); this.learningPath = learningPathData; diff --git a/scripts/count_queries.py b/scripts/count_queries.py index dc65881c..48167c3b 100644 --- a/scripts/count_queries.py +++ b/scripts/count_queries.py @@ -11,8 +11,7 @@ os.environ.setdefault("IT_APP_ENVIRONMENT", "development") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.base") django.setup() -from vbv_lernwelt.learnpath.models import LearningPath -from vbv_lernwelt.learnpath.serializers import LearningPathSerializer +from wagtail.models import Page def main(): @@ -22,9 +21,9 @@ def main(): from django.db import reset_queries reset_queries() - learning_path = LearningPath.objects.filter(slug='versicherungsvermittlerin', locale__language_code='de-CH').first() + page = Page.objects.get(slug='versicherungsvermittlerin', locale__language_code='de-CH') + serializer = page.specific.get_serializer_class()(page.specific) - serializer = LearningPathSerializer(learning_path) print(serializer.data) print(len(json.dumps(serializer.data))) print(len(connection.queries)) diff --git a/server/config/settings/base.py b/server/config/settings/base.py index 176f7ba0..dcd70113 100644 --- a/server/config/settings/base.py +++ b/server/config/settings/base.py @@ -486,31 +486,28 @@ ALLOWED_HOSTS = env.list( # CACHES CACHES = { "default": { - "BACKEND": env( - "IT_DJANGO_CACHE_BACKEND", - default="django.core.cache.backends.db.DatabaseCache", - ), + "BACKEND": env("IT_DJANGO_CACHE_BACKEND", default="django.core.cache.backends.db.DatabaseCache"), "LOCATION": env("IT_DJANGO_CACHE_LOCATION", default="django_cache_table"), - } + }, } if "django_redis.cache.RedisCache" in env("IT_DJANGO_CACHE_BACKEND", default=""): CACHES = { "default": { - "BACKEND": env( - "IT_DJANGO_CACHE_BACKEND", - default="django.core.cache.backends.db.DatabaseCache", - ), - "LOCATION": env("IT_DJANGO_CACHE_LOCATION", default="django_cache_table"), + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": env("IT_DJANGO_CACHE_LOCATION"), "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", - # Mimicing memcache behavior. - # https://github.com/jazzband/django-redis#memcached-exceptions-behavior "IGNORE_EXCEPTIONS": True, }, - } + }, } +CACHES["learning_path_cache"] = { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "django_cache_learning_path", +} + # OAuth/OpenId Connect OAUTH = { diff --git a/server/vbv_lernwelt/learnpath/models.py b/server/vbv_lernwelt/learnpath/models.py index 0e3d1c57..f2d8344b 100644 --- a/server/vbv_lernwelt/learnpath/models.py +++ b/server/vbv_lernwelt/learnpath/models.py @@ -29,6 +29,12 @@ class LearningPath(Page): def __str__(self): return f"{self.title}" + @classmethod + def get_serializer_class(cls): + return get_it_serializer_class( + cls, ['id', 'title', 'slug', 'type', 'translation_key', 'children'] + ) + class Topic(Page): # title = models.TextField(default='') diff --git a/server/vbv_lernwelt/learnpath/serializer_helpers.py b/server/vbv_lernwelt/learnpath/serializer_helpers.py index 5ebfda78..73da6ade 100644 --- a/server/vbv_lernwelt/learnpath/serializer_helpers.py +++ b/server/vbv_lernwelt/learnpath/serializer_helpers.py @@ -25,11 +25,10 @@ class ItBaseSerializer(wagtail_serializers.BaseSerializer): super().__init__(*args, **kwargs) def get_children(self, obj): - if self.descendants: - children = _get_children(self.descendants, obj) - return [c.specific.get_serializer_class()(c.specific, descendants=self.descendants).data for c in children] - else: - return [c.specific.get_serializer_class()(c.specific).data for c in obj.get_children().specific()] + if not self.descendants: + self.descendants = [p for p in obj.get_descendants().specific()] + children = _get_children(self.descendants, obj) + return [c.specific.get_serializer_class()(c.specific, descendants=self.descendants).data for c in children] def _get_descendants(pages, obj): diff --git a/server/vbv_lernwelt/learnpath/serializers.py b/server/vbv_lernwelt/learnpath/serializers.py index ae4dadd9..e69de29b 100644 --- a/server/vbv_lernwelt/learnpath/serializers.py +++ b/server/vbv_lernwelt/learnpath/serializers.py @@ -1,21 +0,0 @@ -from rest_framework import serializers - -from vbv_lernwelt.learnpath.models import Circle, LearningPath -from vbv_lernwelt.learnpath.serializer_helpers import get_it_serializer_class, _get_children - - -class LearningPathSerializer(get_it_serializer_class(LearningPath, [])): - children = serializers.SerializerMethodField() - - meta_fields = [] - - def get_children(self, obj): - descendants = [p for p in obj.get_descendants().specific()] - return [c.specific.get_serializer_class()(c.specific, descendants=descendants).data for c in _get_children(descendants, obj)] - - def get_meta_label(self, obj): - return obj._meta.label - - class Meta: - model = Circle - fields = ['id', 'title', 'slug', 'type', 'translation_key', 'children'] diff --git a/server/vbv_lernwelt/learnpath/signals.py b/server/vbv_lernwelt/learnpath/signals.py new file mode 100644 index 00000000..48369730 --- /dev/null +++ b/server/vbv_lernwelt/learnpath/signals.py @@ -0,0 +1,16 @@ +import structlog +from django.core.cache import caches +from django.db.models.signals import post_delete, post_save +from wagtail.models import Page + +logger = structlog.get_logger(__name__) + + +def invalidate_learning_path_cache(sender, **kwargs): + logger.debug('invalidate learning_path_cache', label='learning_path_cache') + caches['learning_path_cache'].clear() + + +for subclass in Page.__subclasses__(): + post_save.connect(invalidate_learning_path_cache, subclass) + post_delete.connect(invalidate_learning_path_cache, subclass) diff --git a/server/vbv_lernwelt/learnpath/urls.py b/server/vbv_lernwelt/learnpath/urls.py index 5b470619..2d12205e 100644 --- a/server/vbv_lernwelt/learnpath/urls.py +++ b/server/vbv_lernwelt/learnpath/urls.py @@ -1,10 +1,8 @@ from django.urls import path, re_path -from .views import circle_view, generate_web_component_icons -from .views import learningpath_view +from .views import generate_web_component_icons, page_api_view urlpatterns = [ - path(r"api/circle//", circle_view, name="circle_view"), - path(r"api/learningpath//", learningpath_view, name="learningpath_view"), + path(r"api/page//", page_api_view, name="page_api_view"), re_path(r"icons/$", generate_web_component_icons, name="generate_web_component_icons"), ] diff --git a/server/vbv_lernwelt/learnpath/views.py b/server/vbv_lernwelt/learnpath/views.py index 962bc23c..c1ea8f26 100644 --- a/server/vbv_lernwelt/learnpath/views.py +++ b/server/vbv_lernwelt/learnpath/views.py @@ -4,18 +4,19 @@ from pathlib import Path from django.conf import settings from django.shortcuts import render +from django.views.decorators.cache import cache_page from rest_framework.decorators import api_view from rest_framework.response import Response +from wagtail.models import Page from vbv_lernwelt.core.middleware.auth import django_view_authentication_exempt -from vbv_lernwelt.learnpath.models import Circle, LearningPath -from vbv_lernwelt.learnpath.serializers import LearningPathSerializer @api_view(['GET']) -def circle_view(request, slug): - circle = Circle.objects.get(slug=slug) - serializer = Circle.get_serializer_class()(circle) +@cache_page(60 * 60 * 8, cache="learning_path_cache") +def page_api_view(request, slug): + page = Page.objects.get(slug=slug, locale__language_code='de-CH') + serializer = page.specific.get_serializer_class()(page.specific) return Response(serializer.data) @@ -38,9 +39,3 @@ def generate_web_component_icons(request): context={'svg_files': svg_files}, content_type="application/javascript" ) - -@api_view(['GET']) -def learningpath_view(request, slug): - learning_path = LearningPath.objects.get(slug=slug, locale__language_code='de-CH') - serializer = LearningPathSerializer(learning_path) - return Response(serializer.data) From 724b8a8cb16dbe83b68ef69c19a595f339c24ae5 Mon Sep 17 00:00:00 2001 From: Daniel Egger Date: Wed, 6 Jul 2022 09:19:34 +0200 Subject: [PATCH 03/32] Add convenience features --- client/src/App.vue | 6 + client/src/components/MainNavigationBar.vue | 243 ++++++++++-------- .../src/components/circle/CircleDiagram.vue | 26 +- client/src/router/guards.ts | 9 +- client/src/router/index.ts | 28 +- client/src/stores/app.ts | 8 +- client/src/stores/learningPath.ts | 2 - client/src/stores/user.ts | 4 + client/src/views/CircleView.vue | 9 +- 9 files changed, 177 insertions(+), 158 deletions(-) diff --git a/client/src/App.vue b/client/src/App.vue index 10f11ea1..94698338 100644 --- a/client/src/App.vue +++ b/client/src/App.vue @@ -11,6 +11,12 @@ import * as log from 'loglevel'; import MainNavigationBar from '@/components/MainNavigationBar.vue'; import Footer from '@/components/Footer.vue'; +import {onMounted} from 'vue'; log.debug('App created'); + +onMounted(() => { + log.debug('App mounted'); +}); + diff --git a/client/src/components/MainNavigationBar.vue b/client/src/components/MainNavigationBar.vue index 3ddaa904..f922fed2 100644 --- a/client/src/components/MainNavigationBar.vue +++ b/client/src/components/MainNavigationBar.vue @@ -1,12 +1,12 @@ diff --git a/client/src/components/circle/CircleDiagram.vue b/client/src/components/circle/CircleDiagram.vue index dcb2f49b..1eb10a51 100644 --- a/client/src/components/circle/CircleDiagram.vue +++ b/client/src/components/circle/CircleDiagram.vue @@ -5,20 +5,6 @@ import * as _ from 'underscore' import {useCircleStore} from '@/stores/circle'; import * as log from 'loglevel'; -const props = defineProps<{ - width: { - default: 500, - type: number, - required: false - - }, - height: { - default: 500, - type: number, - required: false - }, -}>() - const circleStore = useCircleStore(); function someFinished(learningSequence) { @@ -70,12 +56,12 @@ const blue900 = '#00224D', sky400 = '#72CAFF', sky500 = '#41B5FA' -function render() { - const width = 450, //props.width, - height = 450, //props.height, - radius: number = Math.min(width, height) / 2.4, - arrowStrokeWidth = 2 +const width = 450 +const height = 450 +const radius = Math.min(width, height) / 2.4 +function render() { + const arrowStrokeWidth = 2 const svg = d3.select('.circle-visualization') .attr('viewBox', `0 0 ${width} ${height}`) @@ -224,6 +210,8 @@ function render() { + + diff --git a/client/src/router/guards.ts b/client/src/router/guards.ts index 80596c46..9fe06bfd 100644 --- a/client/src/router/guards.ts +++ b/client/src/router/guards.ts @@ -1,14 +1,13 @@ -import type {NavigationGuardWithThis, RouteLocationNormalized} from 'vue-router'; -import {useUserStore} from '@/stores/user'; - +import type { NavigationGuardWithThis, RouteLocationNormalized } from 'vue-router' +import { useUserStore } from '@/stores/user' export const updateLoggedIn: NavigationGuardWithThis = (_to) => { const loggedIn = getCookieValue('loginStatus') === 'true' const userStore = useUserStore() - userStore.$patch({loggedIn}); + userStore.$patch({ loggedIn }) if (loggedIn && !userStore.email) { - userStore.fetchUser(); + userStore.fetchUser() } } diff --git a/client/src/router/index.ts b/client/src/router/index.ts index 93842418..e6683230 100644 --- a/client/src/router/index.ts +++ b/client/src/router/index.ts @@ -1,7 +1,8 @@ -import {createRouter, createWebHistory} from 'vue-router' -import CockpitView from '@/views/CockpitView.vue'; -import LoginView from '@/views/LoginView.vue'; -import {redirectToLoginIfRequired, updateLoggedIn} from '@/router/guards'; +import { createRouter, createWebHistory } from 'vue-router' +import CockpitView from '@/views/CockpitView.vue' +import LoginView from '@/views/LoginView.vue' +import { redirectToLoginIfRequired, updateLoggedIn } from '@/router/guards' +import { useAppStore } from '@/stores/app' const router = createRouter({ history: createWebHistory(import.meta.env.BASE_URL), @@ -11,8 +12,8 @@ const router = createRouter({ component: LoginView, meta: { // no login required -> so `public === true` - public: true - } + public: true, + }, }, { path: '/', @@ -38,28 +39,33 @@ const router = createRouter({ { path: '/learningpath/:learningPathSlug', component: () => import('../views/LearningPathView.vue'), - props: true + props: true, }, { path: '/circle/:circleSlug', component: () => import('../views/CircleView.vue'), - props: true + props: true, }, { path: '/styleguide', component: () => import('../views/StyelGuideView.vue'), meta: { - public: true - } + public: true, + }, }, { path: '/:pathMatch(.*)*', component: () => import('../views/404View.vue'), }, - ] + ], }) router.beforeEach(updateLoggedIn) router.beforeEach(redirectToLoginIfRequired) +router.afterEach((to, from) => { + const appStore = useAppStore(); + appStore.routingFinished = true; +}); + export default router diff --git a/client/src/stores/app.ts b/client/src/stores/app.ts index 473c102a..8e4bec05 100644 --- a/client/src/stores/app.ts +++ b/client/src/stores/app.ts @@ -1,13 +1,17 @@ -import {defineStore} from 'pinia' +import { defineStore } from 'pinia' export type AppState = { - showMainNavigationBar: boolean; + userLoaded: boolean + routingFinished: boolean + showMainNavigationBar: boolean } export const useAppStore = defineStore({ id: 'app', state: () => ({ showMainNavigationBar: true, + userLoaded: false, + routingFinished: false, } as AppState), getters: { }, diff --git a/client/src/stores/learningPath.ts b/client/src/stores/learningPath.ts index 8422a49d..8759565f 100644 --- a/client/src/stores/learningPath.ts +++ b/client/src/stores/learningPath.ts @@ -54,8 +54,6 @@ export const useLearningPathStore = defineStore({ }) this.learningPath.topics.push(topic); - console.log('#######################'); - console.log(this.learningPath); } return this.learningPath; } catch (error) { diff --git a/client/src/stores/user.ts b/client/src/stores/user.ts index 4881e38c..c756a08a 100644 --- a/client/src/stores/user.ts +++ b/client/src/stores/user.ts @@ -2,6 +2,7 @@ import * as log from 'loglevel'; import {defineStore} from 'pinia' import {itGet, itPost} from '@/fetchHelpers'; +import {useAppStore} from '@/stores/app'; // typed state https://stackoverflow.com/questions/71012513/when-using-pinia-and-typescript-how-do-you-use-an-action-to-set-the-state export type UserState = { @@ -46,11 +47,14 @@ export const useUserStore = defineStore({ } }, fetchUser() { + const appStore = useAppStore(); itGet('/api/core/me/').then((data) => { this.$state = data; this.loggedIn = true; + appStore.userLoaded = true; }).catch(() => { this.loggedIn = false; + appStore.userLoaded = true; }) } } diff --git a/client/src/views/CircleView.vue b/client/src/views/CircleView.vue index bd2e7314..aae3e437 100644 --- a/client/src/views/CircleView.vue +++ b/client/src/views/CircleView.vue @@ -2,12 +2,12 @@ import * as log from 'loglevel'; import LearningSequence from '@/components/circle/LearningSequence.vue'; import CircleOverview from '@/components/circle/CircleOverview.vue'; +import CircleDiagram from '@/components/circle/CircleDiagram.vue'; import LearningContent from '@/components/circle/LearningContent.vue'; import {onMounted} from 'vue' import {useCircleStore} from '@/stores/circle'; import SelfEvaluation from '@/components/circle/SelfEvaluation.vue'; -import CircleDiagram from '@/components/circle/CircleDiagram.vue'; log.debug('CircleView.vue created'); @@ -16,16 +16,16 @@ const props = defineProps<{ }>() const circleStore = useCircleStore(); +circleStore.loadCircle(props.circleSlug); onMounted(async () => { log.info('CircleView.vue mounted'); - await circleStore.loadCircle(props.circleSlug); });