from datetime import date, datetime, time from typing import Any, Dict, List import structlog from django.utils import timezone from openpyxl.reader.excel import load_workbook from vbv_lernwelt.assignment.models import AssignmentType from vbv_lernwelt.core.models import User from vbv_lernwelt.course.consts import COURSE_UK, COURSE_UK_FR, COURSE_UK_IT from vbv_lernwelt.course.models import Course, CourseSession, CourseSessionUser from vbv_lernwelt.course_session.models import ( CourseSessionAssignment, CourseSessionAttendanceCourse, ) from vbv_lernwelt.importer.utils import ( calc_header_tuple_list_from_pyxl_sheet, parse_circle_group_string, try_parse_datetime, ) from vbv_lernwelt.learnpath.models import ( Circle, LearningContentAssignment, LearningContentAttendanceCourse, ) logger = structlog.get_logger(__name__) # it's easier to define the data here, constructing slugs is error-prone and there are some exceptions LP_DATA = { "Kickoff": { "de": { "title": "Kickoff", "slug": "kickoff", "presence_course": "kickoff-lc-präsenzkurs-kickoff", "assignments": [ "kickoff-lc-versicherungswirtschaft", "kickoff-lc-redlichkeitserklärung", "kickoff-lc-reflexion", ], }, "fr": { "title": "Lancement", "slug": "lancement", "presence_course": "lancement-lc-cours-de-présence-lancement", "assignments": [ "lancement-lc-secteur-de-lassurance", "lancement-lc-redlichkeitserklärung", "lancement-lc-réflexion", ], }, "it": { "title": "Introduzione", "slug": "introduzione", "presence_course": "introduzione-lc-corso-di-presenza-introduzione", "assignments": [ "introduzione-lc-settore-assicurativo", "introduzione-lc-redlichkeitserklärung", "introduzione-lc-riflessione", ], }, }, "Basis": { "de": { "title": "Basis", "slug": "basis", "presence_course": "basis-lc-präsenzkurs-basis", "assignments": [ "basis-lc-vorbereitungsauftrag-circle-basis", ], }, "fr": { "title": "Base", "slug": "base", "presence_course": "base-lc-cours-de-présence-base", "assignments": [ "base-lc-mandats-préparatoires-circle-base", ], }, "it": { "title": "Base", "slug": "base", "presence_course": "base-lc-corso-di-presenza-base", "assignments": [ "base-lc-vorbereitungsauftrag-circle-basis", ], }, }, "Fahrzeug": { "de": { "title": "Fahrzeug", "slug": "fahrzeug", "presence_course": "fahrzeug-lc-präsenzkurs-fahrzeug", "assignments": [ "fahrzeug-lc-überprüfen-einer-motorfahrzeug-versicherungspolice", "fahrzeug-lc-fahrzeug-mein-erstes-auto", ], }, "fr": { "title": "Véhicule", "slug": "véhicule", "presence_course": "véhicule-lc-cours-de-présence-véhicule-à-moteur", "assignments": [ "véhicule-lc-vérification-dune-police-dassurance-de-véhicule-à-moteur", "véhicule-lc-véhicule-à-moteur-ma-première-voiture", ], }, "it": { "title": "Veicolo", "slug": "veicolo", "presence_course": "veicolo-lc-corso-di-presenza-veicolo", "assignments": [ "veicolo-lc-verifica-di-una-polizza-di-assicurazione-veicoli-a-motore", "veicolo-lc-veicolo-la-mia-prima-auto", ], }, }, "Haushalt Teil 1": { "de": { "title": "Haushalt Teil 1", "slug": "haushalt-teil-1", "presence_course": "", "assignments": [], }, "fr": { "title": "Habitat partie 1", "slug": "habitat-partie-1", "presence_course": "", "assignments": [], }, "it": {}, }, "Haushalt Teil 2": { "de": { "title": "Haushalt Teil 2", "slug": "haushalt-teil-2", "presence_course": "", "assignments": [], }, "fr": { "title": "Habitat partie 2", "slug": "habitat-partie-2", "presence_course": "", "assignments": [], }, "it": {}, }, } # the request is always, so we cannot rely on the language in the request TRANSLATIONS = { "de": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, "fr": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, "it": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, } T2L_IGNORE_FIELDS = ["Vorname", "Name", "Email", "Sprache", "Durchführungen"] class DataImportError(Exception): pass def create_or_update_user( email: str, first_name: str = "", last_name: str = "", sso_id: str = None, contract_number: str = "", date_of_birth: str = "", ): logger.debug( "create_or_update_user", email=email, first_name=first_name, last_name=last_name, sso_id=sso_id, label="import", ) user = None if sso_id: user_qs = User.objects.filter(sso_id=sso_id) if user_qs.exists(): user = user_qs.first() # use the ID from DBPLAP2 (Lehrvertragsnummer, firstname, lastname, date of birth) if not user and contract_number: user_qs = User.objects.filter( first_name=first_name, last_name=last_name, additional_json_data__Lehrvertragsnummer=contract_number, additional_json_data__Geburtsdatum=date_of_birth, ) if user_qs.exists(): user = user_qs.first() if not user: user_qs = User.objects.filter(email=email) if user_qs.exists(): user = user_qs.first() if not user: # create user user = User(sso_id=sso_id, email=email, username=email) user.email = email user.sso_id = user.sso_id or sso_id user.first_name = first_name or user.first_name user.last_name = last_name or user.last_name user.username = email user.set_unusable_password() user.save() return user def import_course_sessions_from_excel( filename: str, course: Course = None, restrict_language=None ): workbook = load_workbook(filename=filename) sheet = workbook["Schulungen Durchführung"] no_course = course is None tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data(data, ["Klasse", "ID", "Generation", "Region", "Sprache"]) language = data["Sprache"].strip() # this can be removed when the training import (create_course_training_xx) is no longer used if restrict_language and language != restrict_language: continue if no_course: course = get_uk_course(language) create_or_update_course_session( course, data, language, circle_keys=["Kickoff", "Basis", "Fahrzeug"] ) def create_or_update_course_session( course: Course, data: Dict[str, Any], language: str, circle_keys=None, lp_data=LP_DATA, ): """ :param data: the following keys are required to process the data: Generation, Region, Klasse :return: """ logger.debug( "create_or_update_course_session", course=course.title, data=data, label="import", ) if not circle_keys: circle_keys = [] group = data["Klasse"].strip() import_id = data["ID"].strip() generation = str(data["Generation"]).strip() region = data["Region"].strip() title = f"{region} {generation} {group}" cs, _created = CourseSession.objects.get_or_create( title=title, course=course, import_id=import_id ) cs.additional_json_data["import_data"] = data cs.save() cs.title = title cs.generation = generation cs.region = region cs.group = group cs.import_id = import_id cs.save() for circle in circle_keys: circle_data = lp_data[circle][language] attendance_course_lc = LearningContentAttendanceCourse.objects.filter( slug=f"{course.slug}-lp-circle-{circle_data['presence_course']}" ).first() room = data[f"{circle} {TRANSLATIONS[language]['raum']}"] place = data[f"{circle} {TRANSLATIONS[language]['standort']}"] address = data[f"{circle} {TRANSLATIONS[language]['adresse']}"] location = f"{room}, {place}, {address}" start = try_parse_datetime(data[f"{circle} {TRANSLATIONS[language]['start']}"])[ 1 ] end = try_parse_datetime(data[f"{circle} {TRANSLATIONS[language]['ende']}"])[1] if attendance_course_lc: csa, _created = CourseSessionAttendanceCourse.objects.get_or_create( course_session=cs, learning_content=attendance_course_lc ) csa.location = location expert = CourseSessionUser.objects.filter( course_session_id=cs.id, expert__slug=f"{course.slug}-lp-circle-{circle_data['slug']}", role=CourseSessionUser.Role.EXPERT, ).first() if expert: csa.trainer = f"{expert.user.first_name} {expert.user.last_name}" if start: csa.due_date.start = timezone.make_aware(start) if end: csa.due_date.end = timezone.make_aware(end) csa.due_date.save() csa.save() for assignment_slug in circle_data["assignments"]: logger.debug("import", slug=f"{course.slug}-lp-circle-{assignment_slug}") learning_content = LearningContentAssignment.objects.filter( slug=f"{course.slug}-lp-circle-{assignment_slug}" ).first() if learning_content: csa, _created = CourseSessionAssignment.objects.get_or_create( course_session=cs, learning_content=LearningContentAssignment.objects.get( slug=f"{course.slug}-lp-circle-{assignment_slug}" ), ) if ( csa.learning_content.assignment_type == AssignmentType.PREP_ASSIGNMENT.value and start ): csa.submission_deadline.end = timezone.make_aware(start) csa.submission_deadline.save() elif ( csa.learning_content.assignment_type == AssignmentType.CASEWORK.value and end ): csa.submission_deadline.end = timezone.make_aware( start ) + timezone.timedelta(days=30) csa.submission_deadline.save() return cs def validate_row_data(data: Dict[str, any], required_headers: List[str]): logger.debug( "validate_row_data_missing_header", data={"data": data}, label="import", ) for header in required_headers: if str(data.get(header, "")).strip() in ["", "None"]: logger.debug( "validate_row_data_missing_header", data={"data": data, "header": header}, label="import", ) raise DataImportError(f"Missing or empty value for header {header}") def get_uk_course(language: str) -> Course: if language == "fr": course_id = COURSE_UK_FR elif language == "it": course_id = COURSE_UK_IT else: course_id = COURSE_UK return Course.objects.get(id=course_id) def import_trainers_from_excel_for_training( filename: str, language="de", course: Course = None ): workbook = load_workbook(filename=filename) sheet = workbook["Schulungen Trainer"] tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data( data, ["Email", "Vorname", "Name", "Sprache", "Klasse", "Generation"] ) create_or_update_trainer(course, data, language=language) def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"): course_title = course.title if course else "None" logger.debug( "create_or_update_trainer", course=course_title, data=data, label="import", ) user = create_or_update_user( email=data["Email"].lower(), first_name=data["Vorname"], last_name=data["Name"], ) user.language = data["Sprache"] user.save() group = data["Klasse"].strip() # general expert handling import_id = f"{data['Generation'].strip()} {group}" course_session = CourseSession.objects.filter( import_id=import_id, group=group, ).first() if course_session: csu, _created = CourseSessionUser.objects.get_or_create( course_session_id=course_session.id, user_id=user.id ) csu.role = CourseSessionUser.Role.EXPERT csu.save() else: logger.warning( "create_or_update_trainer: course_session not found", import_id=import_id, group=group, label="import", ) if not course and not course_session: logger.warning( "create_or_update_trainer: course_session and course are None", import_id=import_id, group=group, label="import", ) return if not course: course = course_session.course # circle expert handling circle_data = parse_circle_group_string(data["Circles"]) for circle_key in circle_data: circle_name = LP_DATA[circle_key][language]["title"] # print(circle_name, groups) import_id = f"{data['Generation'].strip()} {group}" course_session = CourseSession.objects.filter( import_id=import_id, group=group ).first() circle = Circle.objects.filter( slug=f"{course.slug}-lp-circle-{circle_name.lower()}" ).first() if course_session and circle: csu = CourseSessionUser.objects.filter( course_session_id=course_session.id, user_id=user.id ).first() if csu: csu.expert.add(circle) csu.save() def import_students_from_excel(filename: str): workbook = load_workbook(filename=filename) sheet = workbook.active tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data( data, [ "Email", "Vorname", "Name", "Sprache", "Durchführungen", "Lehrvertragsnummer", ], ) create_or_update_student(data) def create_or_update_student(data: Dict[str, Any]): logger.debug( "create_or_update_student", data=data, label="import", ) date_of_birth = _get_date_of_birth(data) user = create_or_update_user( email=data["Email"].lower(), first_name=data["Vorname"], last_name=data["Name"], contract_number=data.get("Lehrvertragsnummer", ""), date_of_birth=date_of_birth, ) user.language = data["Sprache"] update_user_json_data(user, data) user.save() # general expert handling import_id = data["Durchführungen"] course_session = CourseSession.objects.filter(import_id=import_id).first() if course_session: csu, _created = CourseSessionUser.objects.get_or_create( course_session_id=course_session.id, user_id=user.id ) csu.save() def _get_date_of_birth(data: Dict[str, Any]) -> str: date_of_birth = data.get("Geburtsdatum", None) if date_of_birth is None: return "" elif date_of_birth is date or date_of_birth is datetime: return date_of_birth.strftime("%d.%m.%Y") elif type(date_of_birth) is str: return date_of_birth def sync_students_from_t2l_excel(filename: str): workbook = load_workbook(filename=filename) sheet = workbook.active tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) sync_students_from_t2l(data | {}) def sync_students_from_t2l(data): date_of_birth = _get_date_of_birth(data) try: user = User.objects.get( first_name=data["Vorname"], last_name=data["Name"], additional_json_data__Lehrvertragsnummer=data["Lehrvertragsnummer"], additional_json_data__Geburtsdatum=date_of_birth, ) except User.DoesNotExist: return # only sync data that is not in our user model for field in T2L_IGNORE_FIELDS: try: del data[field] except KeyError: pass update_user_json_data(user, data) user.save() def update_user_json_data(user: User, data: Dict[str, Any]): user.additional_json_data = user.additional_json_data | sanitize_json_data_input( data ) def sanitize_json_data_input(data: Dict[str, Any]) -> Dict[str, Any]: """ Saving additional_json_data fails if the data contains datetime objects. This is a quick and dirty fix to convert datetime objects to iso strings. """ for key, value in data.items(): if isinstance(value, datetime): data[key] = value.isoformat() elif isinstance(value, date): data[key] = value.isoformat() elif isinstance(value, time): data[key] = value.isoformat() else: data[key] = value return data