Add intermediate backups to migrations

server/books/management/commands/analyze_snapshots.py

@@ -1,7 +1,8 @@
 from django.core.management import BaseCommand
 from django.db.models import Count
 from django.db.models.functions import ExtractYear
-from books.models import Snapshot
+from books.models import Snapshot, ObjectiveGroupSnapshot
+from objectives.models import ObjectiveSnapshot
 
 
 # Query to group by creator's email, count the snapshots, and order by the count
@@ -18,3 +19,18 @@ class Command(BaseCommand):
         for snapshot in snapshots_grouped:
             modified_email = snapshot['creator__email'].split('@')[0] + '@skillbox.ch'
             print(f"Year: {snapshot['year']}, Creator Email: {modified_email}, Count: {snapshot['count']}")
+
+        hidden_objectives_counter = 0
+        custom_objectives_counter = 0
+
+        for snapshot in Snapshot.objects.all():
+            if snapshot.hidden_objectives.count() > 0:
+                hidden_objectives_counter += 1
+
+            if snapshot.custom_objectives.count() > 0:
+                custom_objectives_counter += 1
+
+        print(f"Hidden objectives: {hidden_objectives_counter}")
+        print(f"Custom objectives: {custom_objectives_counter}")
+        print(f"ObjectiveGroupSnapshot objectives: {ObjectiveGroupSnapshot.objects.count()}")
+        print(f"ObjectiveSnapshot objectives: {ObjectiveSnapshot.objects.count()}")

server/books/management/commands/migrate_objective_snapshots.py

@@ -1,5 +1,3 @@
-
-
 import json
 from logging import getLogger
 
@@ -7,110 +5,199 @@ from django.core.management import BaseCommand
 
 from books.management.commands.migrate_objectives_to_content import create_chapter_from_objective_group, \
     create_content_block_from_objective, create_text_in_content_block, create_content_block_snapshot_from_objective, \
-    create_chapter_snapshot_from_objective_group
+    create_chapter_snapshot_from_objective_group, create_content_block_contents
 from books.models import Chapter, ObjectiveGroupSnapshot, ContentBlockSnapshot, Snapshot, ChapterSnapshot
 from books.models import ContentBlock
 from books.models import Module
 from objectives.models import Objective, ObjectiveSnapshot, ObjectiveGroup
 
 logger = getLogger(__name__)
+
+
 class Command(BaseCommand):
     def handle(self, *args, **options):
+        """
+        This command must be run after the migration of objectives to content blocks.
+        Migrate objective snapshots to content blocks
+        - Verlagsinhalte - deafult content blocks are referced in the snapshot by foreign key
+
+        Man muss unterscheiden zwischen, snapshots die nur Verlagslernziele sichtbar und unsichtbar machen.
+
+        Und solchen die auch benutzerdefinierte Lernziele sichtbar und unsichtbar machen.
+
+        Es gibt keine custom objective groups!
+
+        Es gibt keine hidden custom objective_groups
+
+        Case1:
+        - 100% Verlagslernziele, 100% Verlagsgruppe, nicht neue content blocks erstellen, nichts hidden... migration muss nichts machen.
+
+        Case2:
+        - 100% Verlagslernziele, 100% Verlagsgruppe, hidden gruppe, nur verlagsinhalt gruppe verstecken
+
+        Case3:
+        - Bestende gruppe mit benutzerdefinierten lernzielen, - custom content_block snapshot erstellen.
+        - Einzelne verslagslernziele sind versteckt. - custom content_block snapshot erstellen.
+
+        --- user created setzen bei custom content snapshot.
+
+        """
         prefix = "SNAP "
         ContentBlock.objects.filter(title__startswith=prefix).delete()
         Chapter.objects.filter(title__startswith=prefix).delete()
         ContentBlockSnapshot.objects.filter(title__startswith=prefix).delete()
         ChapterSnapshot.objects.filter(chapter__title__startswith=prefix).delete()
 
-
-
         analyze()
+        count = 0
+
+        case1_count = 0
+        case2_count = 0
+        case3_count = 0
 
         createed_content_blocks = 0
-
-        failed_modules = []
-
         visible_objectives_by_ids = {}
 
-        for module in Module.objects.filter():
+        for module in Module.objects.all():  # .filter(title__contains="Politik mitbestimmen"):
-            # try:
+            for snapshot in Snapshot.objects.filter(module=module):
+                for objective_group_snapshot in snapshot.objective_groups.through.objects.filter(objective_group__module=module,snapshot=snapshot):
+                    header = f"{count} {module.title:50} {objective_group_snapshot.objective_group.get_title_display():25} {str(snapshot.creator):40} {objective_group_snapshot.hidden} "
+                    count += 1
+                    objective_group = objective_group_snapshot.objective_group
 
-            snapshots = Snapshot.objects.filter(module=module)
+                    hidden_default_objectives = snapshot.hidden_objectives.filter(
-            module_snapshot_by_id = {}
+                        group=objective_group)
-            #chapter = create_chapter_from_objective_group(module)
+
+                    visible_custom_objectives = snapshot.custom_objectives.filter(snapshot=snapshot, hidden=False,
+                                                                                  group=objective_group_snapshot.objective_group)
+                    group_is_hidden = objective_group_snapshot.hidden
 
 
-            for snapshot in snapshots:
+                    info = f"{hidden_default_objectives.count()} {visible_custom_objectives.count()}"
-                # hier objective snapshots entfernt werden. die werden nicht mehr gebraucht
-                # for objective_group_snapshot in self.objective_groups.through.objects.all():
-                #     if objective_group_snapshot.hidden:
-                #         objective_group_snapshot.objective_group.hidden_for.add(selected_class)
-                default_objectives = Objective.objects.filter(group__module=module, owner__isnull=True)
-                visible_objectives = list(default_objectives)
-                visible_objectives_ids = [objective.id for objective in visible_objectives]
 
-                for hidden_objective in snapshot.hidden_objectives.all():
+                    if (not hidden_default_objectives and not visible_custom_objectives and not group_is_hidden):
-                    visible_objectives = [objective for objective in visible_objectives if hidden_objective.id not in visible_objectives_ids]
+                        # print(f"{info}  Case 1 -  skip")
-                for custom_objective_snapshot in snapshot.custom_objectives.all():
+                        case1_count += 1
-                    visible_objectives.append(custom_objective_snapshot)
+                        break
 
-                # filter for unique texts in objectives
+                    if not hidden_default_objectives and not visible_custom_objectives and group_is_hidden:
-                # TODO: I don't know why this is necessary
+                        print(header + f"Case 2 - {info} hide default content group")
-                objectives_by_texts = {}
+                        case2_count += 1
-                for objective in visible_objectives:
+                        content_block = get_content_block_by_objective_group(objective_group_snapshot,
-                    if objective.text not in objectives_by_texts:
+                                                                             module, False, None)
-                        objectives_by_texts[objective.text] = objective
+                        snapshot.hidden_content_blocks.add(content_block.id)
-                visible_objectives = list(objectives_by_texts.values())
+                        snapshot.save()
 
-                if visible_objectives:
+                    if hidden_default_objectives or visible_custom_objectives:
-                    # make comvinations of objectives unique, this prevents generatino of many duplicated content blocks
+                        print(header + f"Case 3 - {info} create custom content blocks")
-                    visible_objectives_hash = hash([objective.text for objective in visible_objectives].__str__())
+                        case3_count += 1
 
-                    visible_objectives_by_ids[visible_objectives_hash] = visible_objectives
+                        default_objectives = Objective.objects.filter(group=objective_group,
+                                                                      group__module=module,
+                                                                      owner__isnull=True)
+                        visible_default_objectives = [objective for objective in default_objectives if objective.id not in hidden_default_objectives.values_list("id", flat=True)]
+
+                        # Benutzerdefinierte Lernziele
+                        visible_custom_objectives = list(snapshot.custom_objectives.filter(hidden=False))
+
+                        visible_objectives = visible_default_objectives + visible_custom_objectives
+
+                        # filter for unique texts in objectives
+                        # TODO: I don't know why there are duplicated objectives
+                        objectives_by_texts = {}
+                        for objective in visible_objectives:
+                            if objective.text not in objectives_by_texts:
+                                objectives_by_texts[objective.text] = objective
+                        visible_objectives = list(objectives_by_texts.values())
+
+                        if visible_objectives:
+                            # make combinations of objectives unique by text, this prevents generation of many duplicated content blocks
+                            visible_objectives_hash = hash(
+                                [objective.text for objective in visible_objectives].__str__())
+
+                            visible_objectives_by_ids[visible_objectives_hash] = visible_objectives
+
+                        for objectives in visible_objectives_by_ids.values():
+                            print("")
+                            for objective in objectives:
+                                print(f"             Objective: {objective.group} {objective}   {objective.owner}")
+                            print("-")
+
+                        print(f"      visible_objectives_by_ids: {len(visible_objectives_by_ids.items())}")
+
+                        # create custom content blocks with the objectives
+                        created_content_blocks = 0
+                        for objectives in visible_objectives_by_ids.values():
+                            chapter = module.get_first_child()
+                            if "Lernziele" not in chapter.title:
+                                raise Exception(f"Chapter does not contain 'Lernziele' first title is {chapter.title}")
+                    #
+                            # Owner des custom blocks festlegen
+                            custom_content_block_snapshot = create_content_block_snapshot_from_objective(
+                                objectives[0].group, chapter, snapshot,
+                                owner=snapshot.creator,
+                                prefix="SNAP ")
+                            create_text_in_content_block(objectives, custom_content_block_snapshot, get_or_create=True)
+                            created_content_blocks += 1
+                            snapshot.save()
+        print()
+        print(f"Skipped {case1_count} Case 1")
+        print(f"Hidden default content groups {case2_count} Case 2")
+        print(f"Created new content {case3_count} Case 3")
 
 
-        for objectives in visible_objectives_by_ids.values():
+def get_content_block_by_objective_group(objective_group_snapshot, module, user_created: bool, user):
-            print("")
+    # TODO: review qustion, is it necessary to filter for module
-            for objective in objectives:
+    content_block = ContentBlock.objects.filter(
-                print(f"             Objective: {objective.group} {objective}   {objective.owner}")
+        title__contains=objective_group_snapshot.objective_group.get_title_display(),
-            print("-")
+        user_created=user_created, owner=user).descendant_of(module)
+    if content_block.count() > 1:
+        return content_block.first()
 
-        print(f"      visible_objectives_by_ids: {len(visible_objectives_by_ids.items())}")
+    return content_block.first()
 
 
-        # create custom content blocks with the objectives
+def get_visible_default_objectives():
-        createed_content_blocks = 0
+    default_objectives = Objective.objects.filter(group=ObjectiveGroup, group__module=module, owner__isnull=True)
-        for objectives in visible_objectives_by_ids.values():
+    visible_objectives = list(default_objectives)
-            snapshot = objectives[0].group.module.snapshots.first()
+    visible_objectives_ids = [objective.id for objective in visible_objectives]
-            module = objectives[0].group.module
-            # does that work?
-            chapter = module.get_first_child()
-            if "Lernziele" not in chapter.title:
-                raise Exception("Chapter does not contain 'Lernziele'")
-            #chapter = create_chapter_snapshot_from_objective_group(module,  snapshot, prefix="SNAP ")
 
-            # Owner des custom blocks festlegen
+    # Verlags Lernziele
-            custom_content_block = create_content_block_snapshot_from_objective(objectives[0].group, chapter, snapshot,
+    for hidden_objective in snapshot.hidden_objectives.all():
-                                                                       owner=None, prefix="SNAP ")
+        visible_objectives = [objective for objective in visible_objectives if
-            create_text_in_content_block(objectives, custom_content_block)
+                              hidden_objective.id not in visible_objectives_ids]
-            createed_content_blocks += 1
+    return visible_objectives
 
-        print(f"created_content_blocks: {createed_content_blocks}")
+
+def get_content_block_by_objectives(objectives, module, user_created=False, user=None):
+    contents = create_content_block_contents(objectives)
+
+    content_block_qs = ContentBlock.objects.filter(
+        owner=user,
+        user_created=user_created,
+        contents=contents)
+    if content_block_qs.exists():
+        return content_block_qs.first()
+    else:
+        raise Exception("Content block does not exist ")
+
+
+#
+
+# print(f"created_content_blocks: {createed_content_blocks}")
 
 
 def analyze():
     print(f"""
     OjectiveGroups: {ObjectiveGroup.objects.count()}
     Objectives: {Objective.objects.count()}
-    
+
     ObjectiveGroupSnapshots: {ObjectiveGroupSnapshot.objects.count()}
     ObjectivesSnapshots: {ObjectiveSnapshot.objects.count()}
-    
+
-    
+
     ObjectiveGroups: {ObjectiveGroup.objects.filter(objectivegroupsnapshot__isnull=True).count()}
     Objectives: {Objective.objects.filter(objectivesnapshot__isnull=True).count()}
-    
+
     Snapshot: {Snapshot.objects.count()}
     """)
-
-

server/books/management/commands/migrate_objectives_to_content.py

@@ -11,7 +11,6 @@ from objectives.models import ObjectiveSnapshot, Objective, ObjectiveGroup
 
 logger = getLogger(__name__)
 
-
 class Command(BaseCommand):
     def handle(self, *args, **options):
         ContentBlock.objects.filter(title__startswith="TESTOBJECTIVE").delete()
@@ -28,11 +27,11 @@ class Command(BaseCommand):
         # In this way we can reuse content blocks for the same set of objectives
         created_default_content_blocks = {}
 
-        for module in Module.objects.filter(title="Politik mitbestimmen"):
+        for module in Module.objects.all():
             try:
                 chapter = create_chapter_from_objective_group(module)
 
-                for objective_group in module.objective_groups.all():
+                for objective_group in module.objective_groups.all().order_by('title'):
                     default_objectives = list(objective_group.objectives.filter(owner__isnull=True, )
                                               .exclude(objectivesnapshot__isnull=False).order_by('order'))
 
@@ -50,6 +49,9 @@ class Command(BaseCommand):
                     if default_objectives or custom_objectives_by_owner:
                         contentblocks_by_merged_objectives_ids = {}
 
+                        # cor custom objectives iterate over owners,
+                        # - one ownsers custom objectives must not be changed by another owner
+                        # - visibility is set per class
                         for owner, owner_objectives in custom_objectives_by_owner.items():
                             print(f"Owner: {owner}")
                             print(f"      Objectives: ")
@@ -101,6 +103,11 @@ class Command(BaseCommand):
                                     custom_content_block.save_revision().publish()
                                     custom_content_block.save()
 
+                                if objective_group.hidden_for.filter(id=school_class.id).exists():
+                                    default_content_block.hidden_for.add(school_class)
+                                    default_content_block.save_revision().publish()
+                                    default_content_block.save()
+
 
             except ValidationError as e:
                 print(f"Error with module {module}")
@@ -214,10 +221,8 @@ def create_content_block_snapshot_from_objective(objective_group, chapter, snaps
     return content_block_snapshot
 
 
-def create_text_in_content_block(objectives, content_block):
+def create_text_in_content_block(objectives, content_block, get_or_create=False):
     objectives = list(objectives)
-    objective_texts = set([objective.text for objective in objectives])
-
     objective_li = [f"<li>{objective.text} -{objective.owner}- </li>" for objective in objectives if objective.text]
 
     texts = [{'type': 'text_block',
@@ -225,10 +230,30 @@ def create_text_in_content_block(objectives, content_block):
                   'text': f"<ul>  {''.join(str(i) for i in objective_li)} </ul>"
               }}]
 
-    content_block.contents = json.dumps(texts)
+    content_block.contents = create_content_block_contents(objectives)
+
+    if get_or_create:
+        content_block_qs = ContentBlock.objects.filter(title=content_block.title,
+                                    owner=content_block.owner,
+                                    user_created=content_block.user_created,
+                                    contents=content_block.contents)
+        if content_block_qs.exists():
+            content_block = content_block_qs.first()
+
     content_block.save_revision().publish()
     return content_block
 
+def create_content_block_contents(objectives):
+    objectives = list(objectives)
+    objective_li = [f"<li>{objective.text} -{objective.owner}- </li>" for objective in objectives if objective.text]
+
+    texts = [{'type': 'text_block',
+              'value': {
+                  'text': f"<ul>  {''.join(str(i) for i in objective_li)} </ul>"
+              }}]
+
+    contents = json.dumps(texts)
+    return contents
 
 def analyze():
     print(f"""
@@ -247,9 +272,10 @@ def analyze():
 
 
 def clean_snapshots():
+    """ utility function to clean up snapshots """
     emails = ["mia.teacher", "simone.gerber", "pascal.sigg", "dario.aebersold", "steph-teacher"]
     for snapshot in Snapshot.objects.all():
         for email in emails:
             if email in snapshot.creator.email:
-                print(f"Deleting snapshot {email}")
+                print(f"Deleting snapshot of user {email}")
                 snapshot.delete()

server/core/management/commands/reset_all_passwords.py

@@ -1,3 +1,6 @@
+from concurrent.futures import ThreadPoolExecutor
+from datetime import timedelta, datetime
+
 from django.contrib.auth import get_user_model
 from django.core.management import BaseCommand
 
@@ -12,6 +15,23 @@ class Command(BaseCommand):
 
         if result == 'YES':
             users = get_user_model().objects.all()
-            for user in users:
+
-                user.set_password('test')
+            with ThreadPoolExecutor(max_workers=10) as executor:
-                user.save()
+                executor.map(process_user, users)
+
+
+def process_user(usr):
+    # replace domain with skillbox.ch
+    usr.email = usr.email.split('@')[0] + '@skillbox.ch'
+    usr.username = usr.email
+    print(usr.email)
+
+    # make license valid for 1 year
+    now = datetime.now()
+
+    if usr.license_expiry_date:
+        usr.license_expiry_date = now + timedelta(days=365)
+
+    # set password to test
+    usr.set_password('test')
+    usr.save()

server/test_objectives_migrations.sh

@@ -0,0 +1,8 @@
+pg_restore --verbose --clean --no-acl --no-owner -h localhost -U skillbox -d skillbox latest.dump
+python manage.py migrate
+python manage.py reset_all_passwords
+pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-anonymized.dump
+python manage.py migrate_objectives_to_content
+pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-migrated-objectives.dump
+#python manage.py migrate_objective_snapshots
+#pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-migrated-objectives-and-snapshots.dump