ramon
/
vbv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merged in feature/fix-import (pull request #179) 

Feature/fix import

Approved-by: Elia Bieri
This commit is contained in:
Christian Cueni 2023-08-09 14:16:17 +00:00
parent 87db790345 5ed289c912
commit 3f8be0a96b
3 changed files with 314 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

223
server/vbv_lernwelt/importer/services.py
View File

@ -2,27 +2,174 @@ from datetime import date, datetime, time
from typing import Any, Dict, List
import structlog
from django.utils import timezone
from openpyxl.reader.excel import load_workbook
from vbv_lernwelt.assignment.models import AssignmentType
from vbv_lernwelt.core.models import User
from vbv_lernwelt.course.consts import COURSE_UK, COURSE_UK_FR, COURSE_UK_IT
from vbv_lernwelt.course.models import Course, CourseSession, CourseSessionUser
from vbv_lernwelt.course_session.models import CourseSessionAttendanceCourse
from vbv_lernwelt.course_session.models import (
CourseSessionAssignment,
CourseSessionAttendanceCourse,
)
from vbv_lernwelt.importer.utils import (
calc_header_tuple_list_from_pyxl_sheet,
parse_circle_group_string,
try_parse_datetime,
)
from vbv_lernwelt.learnpath.models import Circle, LearningContentAttendanceCourse
from vbv_lernwelt.learnpath.models import (
Circle,
LearningContentAssignment,
LearningContentAttendanceCourse,
)
logger = structlog.get_logger(__name__)
CIRCLE_NAMES = {
"Kickoff": {"de": "Kickoff", "fr": "Lancement", "it": "Introduzione"},
"Basis": {"de": "Basis", "fr": "Base", "it": "Base"},
"Fahrzeug": {"de": "Fahrzeug", "fr": "Véhicule", "it": "Veicolo"},
"Haushalt Teil 1": {"de": "Haushalt Teil 1", "fr": "Habitat partie 1", "it": ""},
"Haushalt Teil 2": {"de": "Haushalt Teil 2", "fr": "Haushalt Teil 2", "it": ""},
# it's easier to define the data here, constructing slugs is error-prone and there are some exceptions
LP_DATA = {
"Kickoff": {
"de": {
"title": "Kickoff",
"slug": "kickoff",
"presence_course": "kickoff-lc-präsenzkurs-kickoff",
"assignments": [
"kickoff-lc-versicherungswirtschaft",
"kickoff-lc-redlichkeitserklärung",
"kickoff-lc-reflexion",
],
},
"fr": {
"title": "Lancement",
"slug": "lancement",
"presence_course": "lancement-lc-cours-de-présence-lancement",
"assignments": [
"lancement-lc-secteur-de-lassurance",
"lancement-lc-redlichkeitserklärung",
"lancement-lc-réflexion",
],
},
"it": {
"title": "Introduzione",
"slug": "introduzione",
"presence_course": "introduzione-lc-corso-di-presenza-introduzione",
"assignments": [
"introduzione-lc-settore-assicurativo",
"introduzione-lc-redlichkeitserklärung",
"introduzione-lc-riflessione",
],
},
},
"Basis": {
"de": {
"title": "Basis",
"slug": "basis",
"presence_course": "basis-lc-präsenzkurs-basis",
"assignments": [
"basis-lc-vorbereitungsauftrag-circle-basis",
],
},
"fr": {
"title": "Base",
"slug": "base",
"presence_course": "base-lc-cours-de-présence-base",
"assignments": [
"base-lc-mandats-préparatoires-circle-base",
],
},
"it": {
"title": "Base",
"slug": "base",
"presence_course": "base-lc-corso-di-presenza-base",
"assignments": [
"base-lc-vorbereitungsauftrag-circle-basis",
],
},
},
"Fahrzeug": {
"de": {
"title": "Fahrzeug",
"slug": "fahrzeug",
"presence_course": "fahrzeug-lc-präsenzkurs-fahrzeug",
"assignments": [
"fahrzeug-lc-überprüfen-einer-motorfahrzeug-versicherungspolice",
"fahrzeug-lc-fahrzeug-mein-erstes-auto",
],
},
"fr": {
"title": "Véhicule",
"slug": "véhicule",
"presence_course": "véhicule-lc-cours-de-présence-véhicule-à-moteur",
"assignments": [
"véhicule-lc-vérification-dune-police-dassurance-de-véhicule-à-moteur",
"véhicule-lc-véhicule-à-moteur-ma-première-voiture",
],
},
"it": {
"title": "Veicolo",
"slug": "veicolo",
"presence_course": "veicolo-lc-corso-di-presenza-veicolo",
"assignments": [
"veicolo-lc-verifica-di-una-polizza-di-assicurazione-veicoli-a-motore",
"veicolo-lc-veicolo-la-mia-prima-auto",
],
},
},
"Haushalt Teil 1": {
"de": {
"title": "Haushalt Teil 1",
"slug": "haushalt-teil-1",
"presence_course": "",
"assignments": [],
},
"fr": {
"title": "Habitat partie 1",
"slug": "habitat-partie-1",
"presence_course": "",
"assignments": [],
},
"it": {},
},
"Haushalt Teil 2": {
"de": {
"title": "Haushalt Teil 2",
"slug": "haushalt-teil-2",
"presence_course": "",
"assignments": [],
},
"fr": {
"title": "Habitat partie 2",
"slug": "habitat-partie-2",
"presence_course": "",
"assignments": [],
},
"it": {},
},
}
# the request is always, so we cannot rely on the language in the request
TRANSLATIONS = {
"de": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
"fr": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
"it": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
}
T2L_IGNORE_FIELDS = ["Vorname", "Name", "Email", "Sprache", "Durchführungen"]
@ -85,6 +232,7 @@ def import_course_sessions_from_excel(
):
workbook = load_workbook(filename=filename)
sheet = workbook["Schulungen Durchführung"]
no_course = course is None
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
@ -97,7 +245,7 @@ def import_course_sessions_from_excel(
if restrict_language and language != restrict_language:
continue
if not course:
if no_course:
course = get_uk_course(language)
create_or_update_course_session(
@ -110,6 +258,7 @@ def create_or_update_course_session(
data: Dict[str, Any],
language: str,
circle_keys=None,
lp_data=LP_DATA,
):
"""
:param data: the following keys are required to process the data: Generation, Region, Klasse
@ -148,25 +297,65 @@ def create_or_update_course_session(
cs.save()
for circle in circle_keys:
circle_name = CIRCLE_NAMES[circle][language]
circle_data = lp_data[circle][language]
attendance_course_lc = LearningContentAttendanceCourse.objects.filter(
slug=f"{course.slug}-lp-circle-{circle_name.lower()}-lc-präsenzkurs-{circle_name.lower()}"
slug=f"{course.slug}-lp-circle-{circle_data['presence_course']}"
).first()
room = data[f"{circle} {TRANSLATIONS[language]['raum']}"]
place = data[f"{circle} {TRANSLATIONS[language]['standort']}"]
address = data[f"{circle} {TRANSLATIONS[language]['adresse']}"]
location = f"{room}, {place}, {address}"
start = try_parse_datetime(data[f"{circle} {TRANSLATIONS[language]['start']}"])[
1
]
end = try_parse_datetime(data[f"{circle} {TRANSLATIONS[language]['ende']}"])[1]
if attendance_course_lc:
# update existing data
csa, _created = CourseSessionAttendanceCourse.objects.get_or_create(
course_session=cs, learning_content=attendance_course_lc
)
location = f"{data[f'{circle} Raum']}, {data[f'{circle} Standort']}, {data[f'{circle} Adresse']}"
csa.location = location
expert = CourseSessionUser.objects.filter(
course_session_id=cs.id,
expert__slug=f"{course.slug}-lp-circle-{circle_data['slug']}",
role=CourseSessionUser.Role.EXPERT,
).first()
csa.trainer = ""
csa.due_date.start = try_parse_datetime(data[f"{circle} Start"])[1]
csa.due_date.end = try_parse_datetime(data[f"{circle} Ende"])[1]
if expert:
csa.trainer = f"{expert.user.first_name} {expert.user.last_name}"
if start:
csa.due_date.start = timezone.make_aware(start)
if end:
csa.due_date.end = timezone.make_aware(end)
csa.due_date.save()
csa.save()
for assignment_slug in circle_data["assignments"]:
logger.debug("import", slug=f"{course.slug}-lp-circle-{assignment_slug}")
learning_content = LearningContentAssignment.objects.filter(
slug=f"{course.slug}-lp-circle-{assignment_slug}"
).first()
if learning_content:
csa, _created = CourseSessionAssignment.objects.get_or_create(
course_session=cs,
learning_content=LearningContentAssignment.objects.get(
slug=f"{course.slug}-lp-circle-{assignment_slug}"
),
)
if (
csa.learning_content.assignment_type
== AssignmentType.PREP_ASSIGNMENT.value
and start
):
csa.submission_deadline.end = timezone.make_aware(start)
csa.submission_deadline.save()
return cs
@ -264,7 +453,7 @@ def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"
# circle expert handling
circle_data = parse_circle_group_string(data["Circles"])
for circle_key in circle_data:
circle_name = CIRCLE_NAMES[circle_key][language]
circle_name = LP_DATA[circle_key][language]["title"]
# print(circle_name, groups)
import_id = f"{data['Generation'].strip()} {group}"

107
server/vbv_lernwelt/importer/tests/test_import_course_sessions.py
View File

@ -3,15 +3,19 @@ import os
from django.test import TestCase
from openpyxl.reader.excel import load_workbook
from vbv_lernwelt.core.create_default_users import create_default_users
from vbv_lernwelt.core.models import User
from vbv_lernwelt.course.creators.test_course import create_test_course
from vbv_lernwelt.course.models import CourseSession
from vbv_lernwelt.course.models import CourseSession, CourseSessionUser
from vbv_lernwelt.course_session.models import CourseSessionAttendanceCourse
from vbv_lernwelt.duedate.models import DueDate
from vbv_lernwelt.importer.services import (
create_or_update_course_session,
DataImportError,
validate_row_data,
)
from vbv_lernwelt.importer.utils import calc_header_tuple_list_from_pyxl_sheet
from vbv_lernwelt.learnpath.models import Circle
test_dir = os.path.dirname(os.path.abspath(__file__))
@ -28,9 +32,8 @@ class ImportCourseSessionTestCase(TestCase):
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
print(row)
create_or_update_course_session(
self.course, dict(row), language="de", circle_keys=["Fahrzeug"]
self.course, dict(row), language="de", circle_keys=["Kickoff"]
)
self.assertEqual(CourseSession.objects.count(), 6)
@ -38,23 +41,24 @@ class ImportCourseSessionTestCase(TestCase):
class CreateOrUpdateCourseSessionTestCase(TestCase):
def setUp(self):
create_default_users()
self.course = create_test_course(include_vv=False)
def test_create_course_session(self):
row = [
("ID", "DE 2023 A"),
("ID", "AG 2023 A"),
("Generation", 2023),
("Region", "Deutschschweiz"),
("Region", "Aargau"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 13:30"),
("Fahrzeug Ende", "06.06.2023, 15:00"),
(
"Fahrzeug Raum",
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_N2I5YzViZTQtYTM2Ny00OTYwLTgzNzAtYWI4OTQzODcxNTlj%40thread.v2/0?context=%7b%22Tid%22%3a%22fedd03c8-a756-4803-8f27-0db8f7c488f2%22%2c%22Oid%22%3a%22f92e6382-3884-4e71-a2fd-b305a75d9812%22%7d",
"E64",
),
("Fahrzeug Standort", None),
("Fahrzeug Adresse", None),
("Fahrzeug Standort", "HKV Aarau"),
("Fahrzeug Adresse", "Bahnhofstrasse 460, 5001, Aarau"),
]
data = dict(row)
@ -63,10 +67,10 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
self.course, data, language="de", circle_keys=["Fahrzeug"]
)
self.assertEqual(cs.import_id, "DE 2023 A")
self.assertEqual(cs.title, "Deutschschweiz 2023 A")
self.assertEqual(cs.import_id, "AG 2023 A")
self.assertEqual(cs.title, "Aargau 2023 A")
self.assertEqual(cs.generation, "2023")
self.assertEqual(cs.region, "Deutschschweiz")
self.assertEqual(cs.region, "Aargau")
self.assertEqual(cs.group, "A")
attendance_course = CourseSessionAttendanceCourse.objects.first()
@ -76,29 +80,28 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
self.assertEqual(
attendance_course.due_date.end.isoformat(), "2023-06-06T13:00:00+00:00"
)
self.assertEqual(
f"E64, HKV Aarau, Bahnhofstrasse 460, 5001, Aarau",
attendance_course.location,
)
self.assertEqual("", attendance_course.trainer)
self.assertEqual(4, DueDate.objects.count())
def test_update_course_session(self):
cs = CourseSession.objects.create(
course_id=self.course.id,
title="Deutschschweiz 2023 A",
import_id="DE 2023",
group="A",
)
row = [
("ID", "DE 2023"),
("ID", "AG 2023 A"),
("Generation", 2023),
("Region", "Deutschschweiz"),
("Region", "Aargau"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 13:30"),
("Fahrzeug Ende", "06.06.2023, 15:00"),
(
"Fahrzeug Raum",
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_N2I5YzViZTQtYTM2Ny00OTYwLTgzNzAtYWI4OTQzODcxNTlj%40thread.v2/0?context=%7b%22Tid%22%3a%22fedd03c8-a756-4803-8f27-0db8f7c488f2%22%2c%22Oid%22%3a%22f92e6382-3884-4e71-a2fd-b305a75d9812%22%7d",
"E64",
),
("Fahrzeug Standort", None),
("Fahrzeug Adresse", None),
("Fahrzeug Standort", "HKV Aarau"),
("Fahrzeug Adresse", "Bahnhofstrasse 460, 5001, Aarau"),
]
data = dict(row)
@ -107,21 +110,59 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
self.course, data, language="de", circle_keys=["Fahrzeug"]
)
trainer1 = User.objects.get(email="test-trainer1@example.com")
csu = CourseSessionUser.objects.create(
course_session=cs,
user=trainer1,
role=CourseSessionUser.Role.EXPERT,
)
csu.expert.add(Circle.objects.get(slug="test-lehrgang-lp-circle-fahrzeug"))
update_row = [
("ID", "AG 2023 A"),
("Generation", 2023),
("Region", "Aargau"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 14:30"),
("Fahrzeug Ende", "06.06.2023, 17:00"),
(
"Fahrzeug Raum",
"E666",
),
("Fahrzeug Standort", "HKV Aarau2"),
("Fahrzeug Adresse", "Bahnhofstrasse 460, 5002, Aarau"),
]
data = dict(update_row)
cs = create_or_update_course_session(
self.course, data, language="de", circle_keys=["Fahrzeug"]
)
self.assertEqual(1, CourseSession.objects.count())
self.assertEqual(cs.import_id, "DE 2023")
self.assertEqual(cs.title, "Deutschschweiz 2023 A")
self.assertEqual(cs.import_id, "AG 2023 A")
self.assertEqual(cs.title, "Aargau 2023 A")
self.assertEqual(cs.generation, "2023")
self.assertEqual(cs.region, "Deutschschweiz")
self.assertEqual(cs.region, "Aargau")
self.assertEqual(cs.group, "A")
attendance_course = CourseSessionAttendanceCourse.objects.first()
self.assertEqual(
attendance_course.due_date.start.isoformat(), "2023-06-06T11:30:00+00:00"
attendance_course.due_date.start.isoformat(), "2023-06-06T12:30:00+00:00"
)
self.assertEqual(
attendance_course.due_date.end.isoformat(), "2023-06-06T13:00:00+00:00"
attendance_course.due_date.end.isoformat(), "2023-06-06T15:00:00+00:00"
)
self.assertEqual(
f"E666, HKV Aarau2, Bahnhofstrasse 460, 5002, Aarau",
attendance_course.location,
)
self.assertEqual(
f"{trainer1.first_name} {trainer1.last_name}", attendance_course.trainer
)
self.assertEqual(4, DueDate.objects.count())
def test_import_course_session_twice(self):
"""
@ -129,19 +170,19 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
`CourseSessionAttendanceCourse` only once
"""
row = [
("ID", "DE 2023 A"),
("ID", "AG 2023 A"),
("Generation", 2023),
("Region", "Deutschschweiz"),
("Region", "Aargau"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 13:30"),
("Fahrzeug Ende", "06.06.2023, 15:00"),
(
"Fahrzeug Raum",
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_N2I5YzViZTQtYTM2Ny00OTYwLTgzNzAtYWI4OTQzODcxNTlj%40thread.v2/0?context=%7b%22Tid%22%3a%22fedd03c8-a756-4803-8f27-0db8f7c488f2%22%2c%22Oid%22%3a%22f92e6382-3884-4e71-a2fd-b305a75d9812%22%7d",
"E64",
),
("Fahrzeug Standort", None),
("Fahrzeug Adresse", None),
("Fahrzeug Standort", "HKV Aarau"),
("Fahrzeug Adresse", "Bahnhofstrasse 460, 5001, Aarau"),
]
data = dict(row)

35
server/vbv_lernwelt/importer/tests/test_import_trainers.py
View File

@ -50,7 +50,7 @@ class CreateOrUpdateTrainerTestCase(TestCase):
import_id="DE 2023 A",
group="A",
)
self.course_session_a = CourseSession.objects.create(
self.course_session_b = CourseSession.objects.create(
course=self.course,
title="Deutschschweiz 2023 B",
import_id="DE 2023 B",
@ -96,3 +96,36 @@ class CreateOrUpdateTrainerTestCase(TestCase):
self.assertEqual(csu.role, CourseSessionUser.Role.EXPERT)
self.assertEqual(csu.user.email, "fabienne.haenni@vbv-afa.ch")
self.assertEqual(csu.expert.all().first().title, "Fahrzeug")
def test_update_trainer(self):
row = [
("Name", "Hänni"),
("Vorname", "Fabienne"),
("Email", "fabienne.haenni@vbv-afa.ch"),
("Sprache", "de"),
("Generation", "DE 2023"),
("Klasse", "A"),
("Circles", "Fahrzeug, Haushalt Teil 1"),
]
create_or_update_trainer(self.course, dict(row))
update_row = [
("Name", "Meier"),
("Vorname", "Fabienne"),
("Email", "fabienne.haenni@vbv-afa.ch"),
("Sprache", "de"),
("Generation", "DE 2023"),
("Klasse", "A"),
("Circles", "Fahrzeug, Haushalt Teil 1"),
]
create_or_update_trainer(self.course, dict(update_row))
csu = CourseSessionUser.objects.all().first()
self.assertEqual(csu.course_session, self.course_session_a)
self.assertEqual(csu.role, CourseSessionUser.Role.EXPERT)
self.assertEqual(csu.user.email, "fabienne.haenni@vbv-afa.ch")
self.assertEqual(csu.expert.all().first().title, "Fahrzeug")
self.assertEqual(csu.user.last_name, "Meier")