Add intermediate backups to migrations
This commit is contained in:
Lorenz Padberg
parent
eef536b801
commit
2b7d8eeda3
|
|
@ -1,7 +1,8 @@
|
|||
from django.core.management import BaseCommand
|
||||
from django.db.models import Count
|
||||
from django.db.models.functions import ExtractYear
|
||||
from books.models import Snapshot
|
||||
from books.models import Snapshot, ObjectiveGroupSnapshot
|
||||
from objectives.models import ObjectiveSnapshot
|
||||
|
||||
|
||||
# Query to group by creator's email, count the snapshots, and order by the count
|
||||
|
|
@ -18,3 +19,18 @@ class Command(BaseCommand):
|
|||
for snapshot in snapshots_grouped:
|
||||
modified_email = snapshot['creator__email'].split('@')[0] + '@skillbox.ch'
|
||||
print(f"Year: {snapshot['year']}, Creator Email: {modified_email}, Count: {snapshot['count']}")
|
||||
|
||||
hidden_objectives_counter = 0
|
||||
custom_objectives_counter = 0
|
||||
|
||||
for snapshot in Snapshot.objects.all():
|
||||
if snapshot.hidden_objectives.count() > 0:
|
||||
hidden_objectives_counter += 1
|
||||
|
||||
if snapshot.custom_objectives.count() > 0:
|
||||
custom_objectives_counter += 1
|
||||
|
||||
print(f"Hidden objectives: {hidden_objectives_counter}")
|
||||
print(f"Custom objectives: {custom_objectives_counter}")
|
||||
print(f"ObjectiveGroupSnapshot objectives: {ObjectiveGroupSnapshot.objects.count()}")
|
||||
print(f"ObjectiveSnapshot objectives: {ObjectiveSnapshot.objects.count()}")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
|
||||
|
||||
import json
|
||||
from logging import getLogger
|
||||
|
||||
|
|
@ -7,110 +5,199 @@ from django.core.management import BaseCommand
|
|||
|
||||
from books.management.commands.migrate_objectives_to_content import create_chapter_from_objective_group, \
|
||||
create_content_block_from_objective, create_text_in_content_block, create_content_block_snapshot_from_objective, \
|
||||
create_chapter_snapshot_from_objective_group
|
||||
create_chapter_snapshot_from_objective_group, create_content_block_contents
|
||||
from books.models import Chapter, ObjectiveGroupSnapshot, ContentBlockSnapshot, Snapshot, ChapterSnapshot
|
||||
from books.models import ContentBlock
|
||||
from books.models import Module
|
||||
from objectives.models import Objective, ObjectiveSnapshot, ObjectiveGroup
|
||||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
"""
|
||||
This command must be run after the migration of objectives to content blocks.
|
||||
Migrate objective snapshots to content blocks
|
||||
- Verlagsinhalte - deafult content blocks are referced in the snapshot by foreign key
|
||||
|
||||
Man muss unterscheiden zwischen, snapshots die nur Verlagslernziele sichtbar und unsichtbar machen.
|
||||
|
||||
Und solchen die auch benutzerdefinierte Lernziele sichtbar und unsichtbar machen.
|
||||
|
||||
Es gibt keine custom objective groups!
|
||||
|
||||
Es gibt keine hidden custom objective_groups
|
||||
|
||||
Case1:
|
||||
- 100% Verlagslernziele, 100% Verlagsgruppe, nicht neue content blocks erstellen, nichts hidden... migration muss nichts machen.
|
||||
|
||||
Case2:
|
||||
- 100% Verlagslernziele, 100% Verlagsgruppe, hidden gruppe, nur verlagsinhalt gruppe verstecken
|
||||
|
||||
Case3:
|
||||
- Bestende gruppe mit benutzerdefinierten lernzielen, - custom content_block snapshot erstellen.
|
||||
- Einzelne verslagslernziele sind versteckt. - custom content_block snapshot erstellen.
|
||||
|
||||
--- user created setzen bei custom content snapshot.
|
||||
|
||||
"""
|
||||
prefix = "SNAP "
|
||||
ContentBlock.objects.filter(title__startswith=prefix).delete()
|
||||
Chapter.objects.filter(title__startswith=prefix).delete()
|
||||
ContentBlockSnapshot.objects.filter(title__startswith=prefix).delete()
|
||||
ChapterSnapshot.objects.filter(chapter__title__startswith=prefix).delete()
|
||||
|
||||
|
||||
|
||||
analyze()
|
||||
count = 0
|
||||
|
||||
case1_count = 0
|
||||
case2_count = 0
|
||||
case3_count = 0
|
||||
|
||||
createed_content_blocks = 0
|
||||
|
||||
failed_modules = []
|
||||
|
||||
visible_objectives_by_ids = {}
|
||||
|
||||
for module in Module.objects.filter():
|
||||
# try:
|
||||
for module in Module.objects.all(): # .filter(title__contains="Politik mitbestimmen"):
|
||||
for snapshot in Snapshot.objects.filter(module=module):
|
||||
for objective_group_snapshot in snapshot.objective_groups.through.objects.filter(objective_group__module=module,snapshot=snapshot):
|
||||
header = f"{count} {module.title:50} {objective_group_snapshot.objective_group.get_title_display():25} {str(snapshot.creator):40} {objective_group_snapshot.hidden} "
|
||||
count += 1
|
||||
objective_group = objective_group_snapshot.objective_group
|
||||
|
||||
snapshots = Snapshot.objects.filter(module=module)
|
||||
module_snapshot_by_id = {}
|
||||
#chapter = create_chapter_from_objective_group(module)
|
||||
hidden_default_objectives = snapshot.hidden_objectives.filter(
|
||||
group=objective_group)
|
||||
|
||||
visible_custom_objectives = snapshot.custom_objectives.filter(snapshot=snapshot, hidden=False,
|
||||
group=objective_group_snapshot.objective_group)
|
||||
group_is_hidden = objective_group_snapshot.hidden
|
||||
|
||||
|
||||
for snapshot in snapshots:
|
||||
# hier objective snapshots entfernt werden. die werden nicht mehr gebraucht
|
||||
# for objective_group_snapshot in self.objective_groups.through.objects.all():
|
||||
# if objective_group_snapshot.hidden:
|
||||
# objective_group_snapshot.objective_group.hidden_for.add(selected_class)
|
||||
default_objectives = Objective.objects.filter(group__module=module, owner__isnull=True)
|
||||
visible_objectives = list(default_objectives)
|
||||
visible_objectives_ids = [objective.id for objective in visible_objectives]
|
||||
info = f"{hidden_default_objectives.count()} {visible_custom_objectives.count()}"
|
||||
|
||||
for hidden_objective in snapshot.hidden_objectives.all():
|
||||
visible_objectives = [objective for objective in visible_objectives if hidden_objective.id not in visible_objectives_ids]
|
||||
for custom_objective_snapshot in snapshot.custom_objectives.all():
|
||||
visible_objectives.append(custom_objective_snapshot)
|
||||
if (not hidden_default_objectives and not visible_custom_objectives and not group_is_hidden):
|
||||
# print(f"{info} Case 1 - skip")
|
||||
case1_count += 1
|
||||
break
|
||||
|
||||
# filter for unique texts in objectives
|
||||
# TODO: I don't know why this is necessary
|
||||
objectives_by_texts = {}
|
||||
for objective in visible_objectives:
|
||||
if objective.text not in objectives_by_texts:
|
||||
objectives_by_texts[objective.text] = objective
|
||||
visible_objectives = list(objectives_by_texts.values())
|
||||
if not hidden_default_objectives and not visible_custom_objectives and group_is_hidden:
|
||||
print(header + f"Case 2 - {info} hide default content group")
|
||||
case2_count += 1
|
||||
content_block = get_content_block_by_objective_group(objective_group_snapshot,
|
||||
module, False, None)
|
||||
snapshot.hidden_content_blocks.add(content_block.id)
|
||||
snapshot.save()
|
||||
|
||||
if visible_objectives:
|
||||
# make comvinations of objectives unique, this prevents generatino of many duplicated content blocks
|
||||
visible_objectives_hash = hash([objective.text for objective in visible_objectives].__str__())
|
||||
if hidden_default_objectives or visible_custom_objectives:
|
||||
print(header + f"Case 3 - {info} create custom content blocks")
|
||||
case3_count += 1
|
||||
|
||||
visible_objectives_by_ids[visible_objectives_hash] = visible_objectives
|
||||
default_objectives = Objective.objects.filter(group=objective_group,
|
||||
group__module=module,
|
||||
owner__isnull=True)
|
||||
visible_default_objectives = [objective for objective in default_objectives if objective.id not in hidden_default_objectives.values_list("id", flat=True)]
|
||||
|
||||
# Benutzerdefinierte Lernziele
|
||||
visible_custom_objectives = list(snapshot.custom_objectives.filter(hidden=False))
|
||||
|
||||
visible_objectives = visible_default_objectives + visible_custom_objectives
|
||||
|
||||
# filter for unique texts in objectives
|
||||
# TODO: I don't know why there are duplicated objectives
|
||||
objectives_by_texts = {}
|
||||
for objective in visible_objectives:
|
||||
if objective.text not in objectives_by_texts:
|
||||
objectives_by_texts[objective.text] = objective
|
||||
visible_objectives = list(objectives_by_texts.values())
|
||||
|
||||
if visible_objectives:
|
||||
# make combinations of objectives unique by text, this prevents generation of many duplicated content blocks
|
||||
visible_objectives_hash = hash(
|
||||
[objective.text for objective in visible_objectives].__str__())
|
||||
|
||||
visible_objectives_by_ids[visible_objectives_hash] = visible_objectives
|
||||
|
||||
for objectives in visible_objectives_by_ids.values():
|
||||
print("")
|
||||
for objective in objectives:
|
||||
print(f" Objective: {objective.group} {objective} {objective.owner}")
|
||||
print("-")
|
||||
|
||||
print(f" visible_objectives_by_ids: {len(visible_objectives_by_ids.items())}")
|
||||
|
||||
# create custom content blocks with the objectives
|
||||
created_content_blocks = 0
|
||||
for objectives in visible_objectives_by_ids.values():
|
||||
chapter = module.get_first_child()
|
||||
if "Lernziele" not in chapter.title:
|
||||
raise Exception(f"Chapter does not contain 'Lernziele' first title is {chapter.title}")
|
||||
#
|
||||
# Owner des custom blocks festlegen
|
||||
custom_content_block_snapshot = create_content_block_snapshot_from_objective(
|
||||
objectives[0].group, chapter, snapshot,
|
||||
owner=snapshot.creator,
|
||||
prefix="SNAP ")
|
||||
create_text_in_content_block(objectives, custom_content_block_snapshot, get_or_create=True)
|
||||
created_content_blocks += 1
|
||||
snapshot.save()
|
||||
print()
|
||||
print(f"Skipped {case1_count} Case 1")
|
||||
print(f"Hidden default content groups {case2_count} Case 2")
|
||||
print(f"Created new content {case3_count} Case 3")
|
||||
|
||||
|
||||
for objectives in visible_objectives_by_ids.values():
|
||||
print("")
|
||||
for objective in objectives:
|
||||
print(f" Objective: {objective.group} {objective} {objective.owner}")
|
||||
print("-")
|
||||
def get_content_block_by_objective_group(objective_group_snapshot, module, user_created: bool, user):
|
||||
# TODO: review qustion, is it necessary to filter for module
|
||||
content_block = ContentBlock.objects.filter(
|
||||
title__contains=objective_group_snapshot.objective_group.get_title_display(),
|
||||
user_created=user_created, owner=user).descendant_of(module)
|
||||
if content_block.count() > 1:
|
||||
return content_block.first()
|
||||
|
||||
print(f" visible_objectives_by_ids: {len(visible_objectives_by_ids.items())}")
|
||||
return content_block.first()
|
||||
|
||||
|
||||
# create custom content blocks with the objectives
|
||||
createed_content_blocks = 0
|
||||
for objectives in visible_objectives_by_ids.values():
|
||||
snapshot = objectives[0].group.module.snapshots.first()
|
||||
module = objectives[0].group.module
|
||||
# does that work?
|
||||
chapter = module.get_first_child()
|
||||
if "Lernziele" not in chapter.title:
|
||||
raise Exception("Chapter does not contain 'Lernziele'")
|
||||
#chapter = create_chapter_snapshot_from_objective_group(module, snapshot, prefix="SNAP ")
|
||||
def get_visible_default_objectives():
|
||||
default_objectives = Objective.objects.filter(group=ObjectiveGroup, group__module=module, owner__isnull=True)
|
||||
visible_objectives = list(default_objectives)
|
||||
visible_objectives_ids = [objective.id for objective in visible_objectives]
|
||||
|
||||
# Owner des custom blocks festlegen
|
||||
custom_content_block = create_content_block_snapshot_from_objective(objectives[0].group, chapter, snapshot,
|
||||
owner=None, prefix="SNAP ")
|
||||
create_text_in_content_block(objectives, custom_content_block)
|
||||
createed_content_blocks += 1
|
||||
# Verlags Lernziele
|
||||
for hidden_objective in snapshot.hidden_objectives.all():
|
||||
visible_objectives = [objective for objective in visible_objectives if
|
||||
hidden_objective.id not in visible_objectives_ids]
|
||||
return visible_objectives
|
||||
|
||||
print(f"created_content_blocks: {createed_content_blocks}")
|
||||
|
||||
def get_content_block_by_objectives(objectives, module, user_created=False, user=None):
|
||||
contents = create_content_block_contents(objectives)
|
||||
|
||||
content_block_qs = ContentBlock.objects.filter(
|
||||
owner=user,
|
||||
user_created=user_created,
|
||||
contents=contents)
|
||||
if content_block_qs.exists():
|
||||
return content_block_qs.first()
|
||||
else:
|
||||
raise Exception("Content block does not exist ")
|
||||
|
||||
|
||||
#
|
||||
|
||||
# print(f"created_content_blocks: {createed_content_blocks}")
|
||||
|
||||
|
||||
def analyze():
|
||||
print(f"""
|
||||
OjectiveGroups: {ObjectiveGroup.objects.count()}
|
||||
Objectives: {Objective.objects.count()}
|
||||
|
||||
|
||||
ObjectiveGroupSnapshots: {ObjectiveGroupSnapshot.objects.count()}
|
||||
ObjectivesSnapshots: {ObjectiveSnapshot.objects.count()}
|
||||
|
||||
|
||||
|
||||
|
||||
ObjectiveGroups: {ObjectiveGroup.objects.filter(objectivegroupsnapshot__isnull=True).count()}
|
||||
Objectives: {Objective.objects.filter(objectivesnapshot__isnull=True).count()}
|
||||
|
||||
|
||||
Snapshot: {Snapshot.objects.count()}
|
||||
""")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from objectives.models import ObjectiveSnapshot, Objective, ObjectiveGroup
|
|||
|
||||
logger = getLogger(__name__)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **options):
|
||||
ContentBlock.objects.filter(title__startswith="TESTOBJECTIVE").delete()
|
||||
|
|
@ -28,11 +27,11 @@ class Command(BaseCommand):
|
|||
# In this way we can reuse content blocks for the same set of objectives
|
||||
created_default_content_blocks = {}
|
||||
|
||||
for module in Module.objects.filter(title="Politik mitbestimmen"):
|
||||
for module in Module.objects.all():
|
||||
try:
|
||||
chapter = create_chapter_from_objective_group(module)
|
||||
|
||||
for objective_group in module.objective_groups.all():
|
||||
for objective_group in module.objective_groups.all().order_by('title'):
|
||||
default_objectives = list(objective_group.objectives.filter(owner__isnull=True, )
|
||||
.exclude(objectivesnapshot__isnull=False).order_by('order'))
|
||||
|
||||
|
|
@ -50,6 +49,9 @@ class Command(BaseCommand):
|
|||
if default_objectives or custom_objectives_by_owner:
|
||||
contentblocks_by_merged_objectives_ids = {}
|
||||
|
||||
# cor custom objectives iterate over owners,
|
||||
# - one ownsers custom objectives must not be changed by another owner
|
||||
# - visibility is set per class
|
||||
for owner, owner_objectives in custom_objectives_by_owner.items():
|
||||
print(f"Owner: {owner}")
|
||||
print(f" Objectives: ")
|
||||
|
|
@ -101,6 +103,11 @@ class Command(BaseCommand):
|
|||
custom_content_block.save_revision().publish()
|
||||
custom_content_block.save()
|
||||
|
||||
if objective_group.hidden_for.filter(id=school_class.id).exists():
|
||||
default_content_block.hidden_for.add(school_class)
|
||||
default_content_block.save_revision().publish()
|
||||
default_content_block.save()
|
||||
|
||||
|
||||
except ValidationError as e:
|
||||
print(f"Error with module {module}")
|
||||
|
|
@ -214,10 +221,8 @@ def create_content_block_snapshot_from_objective(objective_group, chapter, snaps
|
|||
return content_block_snapshot
|
||||
|
||||
|
||||
def create_text_in_content_block(objectives, content_block):
|
||||
def create_text_in_content_block(objectives, content_block, get_or_create=False):
|
||||
objectives = list(objectives)
|
||||
objective_texts = set([objective.text for objective in objectives])
|
||||
|
||||
objective_li = [f"<li>{objective.text} -{objective.owner}- </li>" for objective in objectives if objective.text]
|
||||
|
||||
texts = [{'type': 'text_block',
|
||||
|
|
@ -225,10 +230,30 @@ def create_text_in_content_block(objectives, content_block):
|
|||
'text': f"<ul> {''.join(str(i) for i in objective_li)} </ul>"
|
||||
}}]
|
||||
|
||||
content_block.contents = json.dumps(texts)
|
||||
content_block.contents = create_content_block_contents(objectives)
|
||||
|
||||
if get_or_create:
|
||||
content_block_qs = ContentBlock.objects.filter(title=content_block.title,
|
||||
owner=content_block.owner,
|
||||
user_created=content_block.user_created,
|
||||
contents=content_block.contents)
|
||||
if content_block_qs.exists():
|
||||
content_block = content_block_qs.first()
|
||||
|
||||
content_block.save_revision().publish()
|
||||
return content_block
|
||||
|
||||
def create_content_block_contents(objectives):
|
||||
objectives = list(objectives)
|
||||
objective_li = [f"<li>{objective.text} -{objective.owner}- </li>" for objective in objectives if objective.text]
|
||||
|
||||
texts = [{'type': 'text_block',
|
||||
'value': {
|
||||
'text': f"<ul> {''.join(str(i) for i in objective_li)} </ul>"
|
||||
}}]
|
||||
|
||||
contents = json.dumps(texts)
|
||||
return contents
|
||||
|
||||
def analyze():
|
||||
print(f"""
|
||||
|
|
@ -247,9 +272,10 @@ def analyze():
|
|||
|
||||
|
||||
def clean_snapshots():
|
||||
""" utility function to clean up snapshots """
|
||||
emails = ["mia.teacher", "simone.gerber", "pascal.sigg", "dario.aebersold", "steph-teacher"]
|
||||
for snapshot in Snapshot.objects.all():
|
||||
for email in emails:
|
||||
if email in snapshot.creator.email:
|
||||
print(f"Deleting snapshot {email}")
|
||||
print(f"Deleting snapshot of user {email}")
|
||||
snapshot.delete()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import timedelta, datetime
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.management import BaseCommand
|
||||
|
||||
|
|
@ -12,6 +15,23 @@ class Command(BaseCommand):
|
|||
|
||||
if result == 'YES':
|
||||
users = get_user_model().objects.all()
|
||||
for user in users:
|
||||
user.set_password('test')
|
||||
user.save()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
executor.map(process_user, users)
|
||||
|
||||
|
||||
def process_user(usr):
|
||||
# replace domain with skillbox.ch
|
||||
usr.email = usr.email.split('@')[0] + '@skillbox.ch'
|
||||
usr.username = usr.email
|
||||
print(usr.email)
|
||||
|
||||
# make license valid for 1 year
|
||||
now = datetime.now()
|
||||
|
||||
if usr.license_expiry_date:
|
||||
usr.license_expiry_date = now + timedelta(days=365)
|
||||
|
||||
# set password to test
|
||||
usr.set_password('test')
|
||||
usr.save()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
pg_restore --verbose --clean --no-acl --no-owner -h localhost -U skillbox -d skillbox latest.dump
|
||||
python manage.py migrate
|
||||
python manage.py reset_all_passwords
|
||||
pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-anonymized.dump
|
||||
python manage.py migrate_objectives_to_content
|
||||
pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-migrated-objectives.dump
|
||||
#python manage.py migrate_objective_snapshots
|
||||
#pg_dump -Fc --no-acl -h localhost -U skillbox skillbox > latest-migrated-objectives-and-snapshots.dump
|
||||
Loading…
Reference in New Issue