vbv/server/vbv_lernwelt/importer/services.py

707 lines
21 KiB
Python

from datetime import date, datetime, time
from typing import Any, Dict, List
import structlog
from django.utils import timezone
from openpyxl.reader.excel import load_workbook
from vbv_lernwelt.assignment.models import AssignmentType
from vbv_lernwelt.core.models import User
from vbv_lernwelt.course.consts import COURSE_UK, COURSE_UK_FR, COURSE_UK_IT
from vbv_lernwelt.course.models import Course, CourseSession, CourseSessionUser
from vbv_lernwelt.course_session.models import (
CourseSessionAssignment,
CourseSessionAttendanceCourse,
CourseSessionEdoniqTest,
)
from vbv_lernwelt.importer.utils import (
calc_header_tuple_list_from_pyxl_sheet,
parse_circle_group_string,
try_parse_datetime,
)
from vbv_lernwelt.learnpath.models import (
Circle,
LearningContentAssignment,
LearningContentAttendanceCourse,
LearningContentEdoniqTest,
)
logger = structlog.get_logger(__name__)
# it's easier to define the data here, constructing slugs is error-prone and there are some exceptions
LP_DATA = {
"Kickoff": {
"de": {
"title": "Kickoff",
"slug": "kickoff",
"presence_course": "kickoff-lc-präsenzkurs-kickoff",
"assignments": [
"kickoff-lc-vorbereitungsauftrag",
"kickoff-lc-redlichkeitserklärung",
"kickoff-lc-reflexion",
],
"edoniq_tests": ["kickoff-lc-wissens-und-verständnisfragen"],
},
"fr": {
"title": "Lancement",
"slug": "lancement",
"presence_course": "lancement-lc-cours-de-présence-lancement",
"assignments": [
"lancement-lc-mission-de-préparation",
"lancement-lc-déclaration-de-probité",
"lancement-lc-réflexion",
],
"edoniq_tests": [
"lancement-lc-questions-de-connaissances-et-de-compréhension"
],
},
"it": {
"title": "Introduzione",
"slug": "introduzione",
"presence_course": "introduzione-lc-corso-di-presenza-introduzione",
"assignments": [
"introduzione-lc-incarico-di-preparazione",
"introduzione-lc-dichiarazione-di-onestà",
"introduzione-lc-riflessione",
],
"edoniq_tests": ["introduzione-lc-domande-di-conoscenza-e-di-comprensione"],
},
},
"Basis": {
"de": {
"title": "Basis",
"slug": "basis",
"presence_course": "basis-lc-präsenzkurs-basis",
"assignments": [
"basis-lc-vorbereitungsauftrag-circle-basis",
],
"edoniq_tests": ["basis-lc-wissens-und-verständnisfragen"],
},
"fr": {
"title": "Base",
"slug": "base",
"presence_course": "base-lc-cours-de-présence-base",
"assignments": [
"base-lc-mission-de-préparation",
],
"edoniq_tests": ["base-lc-questions-de-connaissances-et-de-compréhension"],
},
"it": {
"title": "Base",
"slug": "base",
"presence_course": "base-lc-corso-di-presenza-base",
"assignments": [
"base-lc-incarico-di-preparazione",
],
"edoniq_tests": ["base-lc-domande-di-conoscenza-e-di-comprensione"],
},
},
"Fahrzeug": {
"de": {
"title": "Fahrzeug",
"slug": "fahrzeug",
"presence_course": "fahrzeug-lc-präsenzkurs-fahrzeug",
"assignments": [
"fahrzeug-lc-überprüfen-einer-motorfahrzeug-versicherungspolice",
"fahrzeug-lc-fahrzeug-mein-erstes-auto",
],
"edoniq_tests": [],
},
"fr": {
"title": "Véhicule",
"slug": "véhicule",
"presence_course": "véhicule-lc-cours-de-présence-véhicule",
"assignments": [
"véhicule-lc-vérification-dune-police-dassurance-de-véhicule-à-moteur",
"véhicule-lc-véhicule-à-moteur-ma-première-voiture",
],
"edoniq_tests": [],
},
"it": {
"title": "Veicolo",
"slug": "veicolo",
"presence_course": "veicolo-lc-corso-di-presenza",
"assignments": [
"veicolo-lc-verifica-di-una-polizza-di-assicurazione-veicoli-a-motore",
"veicolo-lc-veicolo-la-mia-prima-auto",
],
"edoniq_tests": [],
},
},
"Haushalt Teil 1": {
"de": {
"title": "Haushalt Teil 1",
"slug": "haushalt-teil-1",
"presence_course": "haushalt-teil-1-lc-präsenzkurs-haushalt-1",
"assignments": [],
"edoniq_tests": [],
},
"fr": {
"title": "Ménage partie 1",
"slug": "ménage-partie-1",
"presence_course": "ménage-partie-1-lc-cours-de-présence-ménage-partie-1",
"assignments": [],
},
"it": {
"title": "Economica domestica parte 1",
"slug": "economica-domestica-parte-1",
"presence_course": "economica-domestica-parte-1-lc-corso-di-presenza-economica-domestica-parte-1",
"assignments": [],
"edoniq_tests": [],
},
},
"Haushalt Teil 2": {
"de": {
"title": "Haushalt Teil 2",
"slug": "haushalt-teil-2",
"presence_course": "haushalt-teil-2-lc-präsenzkurs-haushalt-2",
"assignments": [],
"edoniq_tests": [],
},
"fr": {
"title": "Ménage partie 2",
"slug": "ménage-partie-2",
"presence_course": "ménage-partie-2-lc-cours-de-présence-ménage-partie-2",
"assignments": [],
},
"it": {
"title": "Economica domestica parte 2",
"slug": "economica-domestica-parte-2",
"presence_course": "ménage-partie-2-lc-cours-de-présence-ménage-partie-2",
"assignments": [],
"edoniq_tests": [],
},
},
}
# the request is always, so we cannot rely on the language in the request
TRANSLATIONS = {
"de": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
"fr": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
"it": {
"raum": "Raum",
"standort": "Standort",
"adresse": "Adresse",
"start": "Start",
"ende": "Ende",
},
}
T2L_IGNORE_FIELDS = ["Vorname", "Name", "Email", "Sprache", "Durchführungen"]
class DataImportError(Exception):
pass
def create_or_update_user(
email: str,
first_name: str = "",
last_name: str = "",
sso_id: str = None,
contract_number: str = "",
date_of_birth: str = "",
):
logger.debug(
"create_or_update_user",
email=email,
first_name=first_name,
last_name=last_name,
sso_id=sso_id,
label="import",
)
user = None
if sso_id:
user_qs = User.objects.filter(sso_id=sso_id)
if user_qs.exists():
user = user_qs.first()
# use the ID from DBPLAP2 (Lehrvertragsnummer, firstname, lastname, date of birth)
if not user and contract_number:
user_qs = User.objects.filter(
first_name=first_name,
last_name=last_name,
additional_json_data__Lehrvertragsnummer=contract_number,
additional_json_data__Geburtsdatum=date_of_birth,
)
if user_qs.exists():
user = user_qs.first()
if not user:
user_qs = User.objects.filter(email=email)
if user_qs.exists():
user = user_qs.first()
if not user:
# create user
user = User(sso_id=sso_id, email=email, username=email)
user.email = email
user.sso_id = user.sso_id or sso_id
user.first_name = first_name or user.first_name
user.last_name = last_name or user.last_name
user.username = email
user.set_unusable_password()
user.save()
return user
def import_course_sessions_from_excel(
filename: str, course: Course = None, restrict_language=None
):
workbook = load_workbook(filename=filename)
sheet = workbook["Schulungen Durchführung"]
no_course = course is None
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
data = dict(row)
validate_row_data(data, ["Klasse", "ID", "Generation", "Region", "Sprache"])
language = data["Sprache"].strip()
# this can be removed when the training import (create_course_training_xx) is no longer used
if restrict_language and language != restrict_language:
continue
if no_course:
course = get_uk_course(language)
create_or_update_course_session(
course, data, language, circle_keys=["Kickoff", "Basis", "Fahrzeug"]
)
def create_or_update_course_session(
course: Course,
data: Dict[str, Any],
language: str,
circle_keys=None,
lp_data=LP_DATA,
):
"""
:param data: the following keys are required to process the data: Generation, Region, Klasse
:return:
"""
logger.debug(
"create_or_update_course_session",
course=course.title,
data=data,
label="import",
)
if not circle_keys:
circle_keys = []
group = data["Klasse"].strip()
import_id = data["ID"].strip()
generation = str(data["Generation"]).strip()
region = data["Region"].strip()
title = f"{region} {generation} {group}"
cs, _created = CourseSession.objects.get_or_create(
title=title, course=course, import_id=import_id
)
cs.additional_json_data["import_data"] = data
cs.save()
cs.title = title
cs.generation = generation
cs.region = region
cs.group = group
cs.import_id = import_id
cs.save()
for circle in circle_keys:
circle_data = lp_data[circle][language]
logger.debug("import", data=circle_data)
attendance_course_lc = LearningContentAttendanceCourse.objects.filter(
slug=f"{course.slug}-lp-circle-{circle_data['presence_course']}"
).first()
room = data[f"{circle} {TRANSLATIONS[language]['raum']}"]
place = data[f"{circle} {TRANSLATIONS[language]['standort']}"]
address = data[f"{circle} {TRANSLATIONS[language]['adresse']}"]
location = f"{room}, {place}, {address}"
presence_day_start = try_parse_datetime(
data[f"{circle} {TRANSLATIONS[language]['start']}"]
)[1]
presence_day_end = try_parse_datetime(
data[f"{circle} {TRANSLATIONS[language]['ende']}"]
)[1]
if attendance_course_lc:
create_or_update_course_session_attendance(
cs,
attendance_course_lc,
course.slug,
circle_data["slug"],
location,
presence_day_start,
presence_day_end,
)
for assignment_slug in circle_data["assignments"]:
create_or_update_course_session_assignment(
cs, course.slug, assignment_slug, presence_day_start, presence_day_end
)
for test_slug in circle_data["edoniq_tests"]:
create_or_update_course_session_edoniq_test(
cs, course.slug, test_slug, presence_day_start
)
return cs
def create_or_update_course_session_attendance(
cs: CourseSession,
attendance_course_lc: LearningContentAttendanceCourse,
course_slug: str,
circle_slug: str,
location: str,
start: datetime,
end: datetime,
):
logger.debug(
"create_or_update_course_session_attendance",
slug=f"{course_slug}-lp-circle-{circle_slug}",
start=start,
end=end,
)
csa, _created = CourseSessionAttendanceCourse.objects.get_or_create(
course_session=cs, learning_content=attendance_course_lc
)
# trigger save to update due date
csa.save()
csa.location = location
expert = CourseSessionUser.objects.filter(
course_session_id=cs.id,
expert__slug=f"{course_slug}-lp-circle-{circle_slug}",
role=CourseSessionUser.Role.EXPERT,
).first()
if expert:
csa.trainer = f"{expert.user.first_name} {expert.user.last_name}"
if start:
csa.due_date.start = timezone.make_aware(start)
if end:
csa.due_date.end = timezone.make_aware(end)
csa.due_date.save()
csa.save()
def create_or_update_course_session_assignment(
cs: CourseSession,
course_slug: str,
assignment_slug: str,
start: datetime,
end: datetime,
):
logger.debug("import", slug=f"{course_slug}-lp-circle-{assignment_slug}")
learning_content = LearningContentAssignment.objects.filter(
slug=f"{course_slug}-lp-circle-{assignment_slug}"
).first()
if learning_content:
csa, _created = CourseSessionAssignment.objects.get_or_create(
course_session=cs,
learning_content=LearningContentAssignment.objects.get(
slug=f"{course_slug}-lp-circle-{assignment_slug}"
),
)
# trigger save to update due date
csa.save()
if (
csa.learning_content.assignment_type == AssignmentType.PREP_ASSIGNMENT.value
and start
):
csa.submission_deadline.start = timezone.make_aware(start)
csa.submission_deadline.end = None
csa.submission_deadline.save()
elif (
csa.learning_content.assignment_type == AssignmentType.CASEWORK.value
and end
):
csa.submission_deadline.start = timezone.make_aware(
start
) + timezone.timedelta(days=30)
csa.submission_deadline.end = None
csa.submission_deadline.save()
def create_or_update_course_session_edoniq_test(
cs: CourseSession, course_slug: str, test_slug: str, start: datetime
):
learning_content = LearningContentEdoniqTest.objects.filter(
slug=f"{course_slug}-lp-circle-{test_slug}"
).first()
if learning_content:
cset, _created = CourseSessionEdoniqTest.objects.get_or_create(
course_session=cs, learning_content=learning_content
)
# trigger save to update due date
cset.save()
cset.deadline.start = timezone.make_aware(start) + timezone.timedelta(days=10)
cset.deadline.end = None
cset.deadline.save()
def validate_row_data(data: Dict[str, any], required_headers: List[str]):
logger.debug(
"validate_row_data_missing_header",
data={"data": data},
label="import",
)
for header in required_headers:
if str(data.get(header, "")).strip() in ["", "None"]:
logger.debug(
"validate_row_data_missing_header",
data={"data": data, "header": header},
label="import",
)
raise DataImportError(f"Missing or empty value for header {header}")
def get_uk_course(language: str) -> Course:
if language == "fr":
course_id = COURSE_UK_FR
elif language == "it":
course_id = COURSE_UK_IT
else:
course_id = COURSE_UK
return Course.objects.get(id=course_id)
def import_trainers_from_excel_for_training(
filename: str, language="de", course: Course = None
):
workbook = load_workbook(filename=filename)
sheet = workbook["Schulungen Trainer"]
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
data = dict(row)
validate_row_data(
data, ["Email", "Vorname", "Name", "Sprache", "Klasse", "Generation"]
)
create_or_update_trainer(course, data, language=language)
def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"):
course_title = course.title if course else "None"
logger.debug(
"create_or_update_trainer",
course=course_title,
data=data,
label="import",
)
user = create_or_update_user(
email=data["Email"].lower(),
first_name=data["Vorname"],
last_name=data["Name"],
)
user.language = data["Sprache"]
user.save()
group = data["Klasse"].strip()
# general expert handling
import_id = f"{data['Generation'].strip()} {group}"
course_session = CourseSession.objects.filter(
import_id=import_id,
group=group,
).first()
if course_session:
csu, _created = CourseSessionUser.objects.get_or_create(
course_session_id=course_session.id, user_id=user.id
)
csu.role = CourseSessionUser.Role.EXPERT
csu.save()
else:
logger.warning(
"create_or_update_trainer: course_session not found",
import_id=import_id,
group=group,
label="import",
)
if not course and not course_session:
logger.warning(
"create_or_update_trainer: course_session and course are None",
import_id=import_id,
group=group,
label="import",
)
return
if not course:
course = course_session.course
# circle expert handling
circle_data = parse_circle_group_string(data["Circles"])
for circle_key in circle_data:
circle_name = LP_DATA[circle_key][language]["title"]
# print(circle_name, groups)
import_id = f"{data['Generation'].strip()} {group}"
course_session = CourseSession.objects.filter(
import_id=import_id, group=group
).first()
circle = Circle.objects.filter(
slug=f"{course.slug}-lp-circle-{circle_name.lower()}"
).first()
if course_session and circle:
csu = CourseSessionUser.objects.filter(
course_session_id=course_session.id, user_id=user.id
).first()
if csu:
csu.expert.add(circle)
csu.save()
def import_students_from_excel(filename: str):
workbook = load_workbook(filename=filename)
sheet = workbook.active
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
data = dict(row)
validate_row_data(
data,
[
"Email",
"Vorname",
"Name",
"Sprache",
"Durchführungen",
"Lehrvertragsnummer",
],
)
create_or_update_student(data)
def create_or_update_student(data: Dict[str, Any]):
logger.debug(
"create_or_update_student",
data=data,
label="import",
)
date_of_birth = _get_date_of_birth(data)
user = create_or_update_user(
email=data["Email"].lower(),
first_name=data["Vorname"],
last_name=data["Name"],
contract_number=data.get("Lehrvertragsnummer", ""),
date_of_birth=date_of_birth,
)
user.language = data["Sprache"]
update_user_json_data(user, data)
user.save()
# general expert handling
import_id = data["Durchführungen"]
course_session = CourseSession.objects.filter(import_id=import_id).first()
if course_session:
csu, _created = CourseSessionUser.objects.get_or_create(
course_session_id=course_session.id, user_id=user.id
)
csu.save()
def _get_date_of_birth(data: Dict[str, Any]) -> str:
date_of_birth = data.get("Geburtsdatum", None)
if date_of_birth is None:
return ""
elif date_of_birth is date or date_of_birth is datetime:
return date_of_birth.strftime("%d.%m.%Y")
elif type(date_of_birth) is str:
return date_of_birth
def sync_students_from_t2l_excel(filename: str):
workbook = load_workbook(filename=filename)
sheet = workbook.active
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
data = dict(row)
sync_students_from_t2l(data | {})
def sync_students_from_t2l(data):
date_of_birth = _get_date_of_birth(data)
try:
user = User.objects.get(
first_name=data["Vorname"],
last_name=data["Name"],
additional_json_data__Lehrvertragsnummer=data["Lehrvertragsnummer"],
additional_json_data__Geburtsdatum=date_of_birth,
)
except User.DoesNotExist:
return
# only sync data that is not in our user model
for field in T2L_IGNORE_FIELDS:
try:
del data[field]
except KeyError:
pass
update_user_json_data(user, data)
user.save()
def update_user_json_data(user: User, data: Dict[str, Any]):
user.additional_json_data = user.additional_json_data | sanitize_json_data_input(
data
)
def sanitize_json_data_input(data: Dict[str, Any]) -> Dict[str, Any]:
"""
Saving additional_json_data fails if the data contains datetime objects.
This is a quick and dirty fix to convert datetime objects to iso strings.
"""
for key, value in data.items():
if isinstance(value, datetime):
data[key] = value.isoformat()
elif isinstance(value, date):
data[key] = value.isoformat()
elif isinstance(value, time):
data[key] = value.isoformat()
else:
data[key] = value
return data