from datetime import date, datetime, time from typing import Any, Dict, List import structlog from django.utils import timezone from openpyxl.reader.excel import load_workbook from vbv_lernwelt.assignment.models import AssignmentType from vbv_lernwelt.core.models import User from vbv_lernwelt.course.consts import COURSE_UK, COURSE_UK_FR, COURSE_UK_IT from vbv_lernwelt.course.models import Course, CourseSession, CourseSessionUser from vbv_lernwelt.course_session.models import ( CourseSessionAssignment, CourseSessionAttendanceCourse, CourseSessionEdoniqTest, ) from vbv_lernwelt.importer.utils import ( calc_header_tuple_list_from_pyxl_sheet, parse_circle_group_string, try_parse_datetime, ) from vbv_lernwelt.learnpath.models import ( Circle, LearningContentAssignment, LearningContentAttendanceCourse, LearningContentEdoniqTest, ) logger = structlog.get_logger(__name__) # it's easier to define the data here, constructing slugs is error-prone and there are some exceptions LP_DATA = { "Kickoff": { "de": { "title": "Kickoff", "slug": "kickoff", "presence_course": "kickoff-lc-präsenzkurs-kickoff", "assignments": [ "kickoff-lc-vorbereitungsauftrag", "kickoff-lc-redlichkeitserklärung", "kickoff-lc-reflexion", ], "edoniq_tests": ["kickoff-lc-wissens-und-verständnisfragen"], }, "fr": { "title": "Lancement", "slug": "lancement", "presence_course": "lancement-lc-cours-de-présence-lancement", "assignments": [ "lancement-lc-mission-de-préparation", "lancement-lc-déclaration-de-probité", "lancement-lc-réflexion", ], "edoniq_tests": [ "lancement-lc-questions-de-connaissances-et-de-compréhension" ], }, "it": { "title": "Introduzione", "slug": "introduzione", "presence_course": "introduzione-lc-corso-di-presenza-introduzione", "assignments": [ "introduzione-lc-incarico-di-preparazione", "introduzione-lc-dichiarazione-di-onestà", "introduzione-lc-riflessione", ], "edoniq_tests": ["introduzione-lc-domande-di-conoscenza-e-di-comprensione"], }, }, "Basis": { "de": { "title": "Basis", "slug": "basis", "presence_course": "basis-lc-präsenzkurs-basis", "assignments": [ "basis-lc-vorbereitungsauftrag-circle-basis", ], "edoniq_tests": ["basis-lc-wissens-und-verständnisfragen"], }, "fr": { "title": "Base", "slug": "base", "presence_course": "base-lc-cours-de-présence-base", "assignments": [ "base-lc-mission-de-préparation", ], "edoniq_tests": ["base-lc-questions-de-connaissances-et-de-compréhension"], }, "it": { "title": "Base", "slug": "base", "presence_course": "base-lc-corso-di-presenza-base", "assignments": [ "base-lc-incarico-di-preparazione", ], "edoniq_tests": ["base-lc-domande-di-conoscenza-e-di-comprensione"], }, }, "Fahrzeug": { "de": { "title": "Fahrzeug", "slug": "fahrzeug", "presence_course": "fahrzeug-lc-präsenzkurs-fahrzeug", "assignments": [ "fahrzeug-lc-überprüfen-einer-motorfahrzeug-versicherungspolice", "fahrzeug-lc-fahrzeug-mein-erstes-auto", ], "edoniq_tests": [], }, "fr": { "title": "Véhicule", "slug": "véhicule", "presence_course": "véhicule-lc-cours-de-présence-véhicule", "assignments": [ "véhicule-lc-vérification-dune-police-dassurance-de-véhicule-à-moteur", "véhicule-lc-véhicule-à-moteur-ma-première-voiture", ], "edoniq_tests": [], }, "it": { "title": "Veicolo", "slug": "veicolo", "presence_course": "veicolo-lc-corso-di-presenza", "assignments": [ "veicolo-lc-verifica-di-una-polizza-di-assicurazione-veicoli-a-motore", "veicolo-lc-veicolo-la-mia-prima-auto", ], "edoniq_tests": [], }, }, "Haushalt Teil 1": { "de": { "title": "Haushalt Teil 1", "slug": "haushalt-teil-1", "presence_course": "haushalt-teil-1-lc-präsenzkurs-haushalt-1", "assignments": [], "edoniq_tests": [], }, "fr": { "title": "Ménage partie 1", "slug": "ménage-partie-1", "presence_course": "ménage-partie-1-lc-cours-de-présence-ménage-partie-1", "assignments": [], }, "it": { "title": "Economica domestica parte 1", "slug": "economica-domestica-parte-1", "presence_course": "economica-domestica-parte-1-lc-corso-di-presenza-economica-domestica-parte-1", "assignments": [], "edoniq_tests": [], }, }, "Haushalt Teil 2": { "de": { "title": "Haushalt Teil 2", "slug": "haushalt-teil-2", "presence_course": "haushalt-teil-2-lc-präsenzkurs-haushalt-2", "assignments": [], "edoniq_tests": [], }, "fr": { "title": "Ménage partie 2", "slug": "ménage-partie-2", "presence_course": "ménage-partie-2-lc-cours-de-présence-ménage-partie-2", "assignments": [], }, "it": { "title": "Economica domestica parte 2", "slug": "economica-domestica-parte-2", "presence_course": "ménage-partie-2-lc-cours-de-présence-ménage-partie-2", "assignments": [], "edoniq_tests": [], }, }, } # the request is always, so we cannot rely on the language in the request TRANSLATIONS = { "de": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, "fr": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, "it": { "raum": "Raum", "standort": "Standort", "adresse": "Adresse", "start": "Start", "ende": "Ende", }, } T2L_IGNORE_FIELDS = ["Vorname", "Name", "Email", "Sprache", "Durchführungen"] class DataImportError(Exception): pass def create_or_update_user( email: str, first_name: str = "", last_name: str = "", sso_id: str = None, contract_number: str = "", date_of_birth: str = "", ): logger.debug( "create_or_update_user", email=email, first_name=first_name, last_name=last_name, sso_id=sso_id, label="import", ) user = None if sso_id: user_qs = User.objects.filter(sso_id=sso_id) if user_qs.exists(): user = user_qs.first() # use the ID from DBPLAP2 (Lehrvertragsnummer, firstname, lastname, date of birth) if not user and contract_number: user_qs = User.objects.filter( first_name=first_name, last_name=last_name, additional_json_data__Lehrvertragsnummer=contract_number, additional_json_data__Geburtsdatum=date_of_birth, ) if user_qs.exists(): user = user_qs.first() if not user: user_qs = User.objects.filter(email=email) if user_qs.exists(): user = user_qs.first() if not user: # create user user = User(sso_id=sso_id, email=email, username=email) user.email = email user.sso_id = user.sso_id or sso_id user.first_name = first_name or user.first_name user.last_name = last_name or user.last_name user.username = email user.set_unusable_password() user.save() return user def import_course_sessions_from_excel( filename: str, course: Course = None, restrict_language=None ): workbook = load_workbook(filename=filename) sheet = workbook["Schulungen Durchführung"] no_course = course is None tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data(data, ["Klasse", "ID", "Generation", "Region", "Sprache"]) language = data["Sprache"].strip() # this can be removed when the training import (create_course_training_xx) is no longer used if restrict_language and language != restrict_language: continue if no_course: course = get_uk_course(language) create_or_update_course_session( course, data, language, circle_keys=["Kickoff", "Basis", "Fahrzeug"] ) def create_or_update_course_session( course: Course, data: Dict[str, Any], language: str, circle_keys=None, lp_data=LP_DATA, ): """ :param data: the following keys are required to process the data: Generation, Region, Klasse :return: """ logger.debug( "create_or_update_course_session", course=course.title, data=data, label="import", ) if not circle_keys: circle_keys = [] group = data["Klasse"].strip() import_id = data["ID"].strip() generation = str(data["Generation"]).strip() region = data["Region"].strip() title = f"{region} {generation} {group}" cs, _created = CourseSession.objects.get_or_create( title=title, course=course, import_id=import_id ) cs.additional_json_data["import_data"] = data cs.save() cs.title = title cs.generation = generation cs.region = region cs.group = group cs.import_id = import_id cs.save() for circle in circle_keys: circle_data = lp_data[circle][language] logger.debug("import", data=circle_data) attendance_course_lc = LearningContentAttendanceCourse.objects.filter( slug=f"{course.slug}-lp-circle-{circle_data['presence_course']}" ).first() room = data[f"{circle} {TRANSLATIONS[language]['raum']}"] place = data[f"{circle} {TRANSLATIONS[language]['standort']}"] address = data[f"{circle} {TRANSLATIONS[language]['adresse']}"] location = f"{room}, {place}, {address}" presence_day_start = try_parse_datetime( data[f"{circle} {TRANSLATIONS[language]['start']}"] )[1] presence_day_end = try_parse_datetime( data[f"{circle} {TRANSLATIONS[language]['ende']}"] )[1] if attendance_course_lc: create_or_update_course_session_attendance( cs, attendance_course_lc, course.slug, circle_data["slug"], location, presence_day_start, presence_day_end, ) for assignment_slug in circle_data["assignments"]: create_or_update_course_session_assignment( cs, course.slug, assignment_slug, presence_day_start, presence_day_end ) for test_slug in circle_data["edoniq_tests"]: create_or_update_course_session_edoniq_test( cs, course.slug, test_slug, presence_day_start ) return cs def create_or_update_course_session_attendance( cs: CourseSession, attendance_course_lc: LearningContentAttendanceCourse, course_slug: str, circle_slug: str, location: str, start: datetime, end: datetime, ): logger.debug( "create_or_update_course_session_attendance", slug=f"{course_slug}-lp-circle-{circle_slug}", start=start, end=end, ) csa, _created = CourseSessionAttendanceCourse.objects.get_or_create( course_session=cs, learning_content=attendance_course_lc ) # trigger save to update due date csa.save() csa.location = location expert = CourseSessionUser.objects.filter( course_session_id=cs.id, expert__slug=f"{course_slug}-lp-circle-{circle_slug}", role=CourseSessionUser.Role.EXPERT, ).first() if expert: csa.trainer = f"{expert.user.first_name} {expert.user.last_name}" if start: csa.due_date.start = timezone.make_aware(start) if end: csa.due_date.end = timezone.make_aware(end) csa.due_date.save() csa.save() def create_or_update_course_session_assignment( cs: CourseSession, course_slug: str, assignment_slug: str, start: datetime, end: datetime, ): logger.debug("import", slug=f"{course_slug}-lp-circle-{assignment_slug}") learning_content = LearningContentAssignment.objects.filter( slug=f"{course_slug}-lp-circle-{assignment_slug}" ).first() if learning_content: csa, _created = CourseSessionAssignment.objects.get_or_create( course_session=cs, learning_content=LearningContentAssignment.objects.get( slug=f"{course_slug}-lp-circle-{assignment_slug}" ), ) # trigger save to update due date csa.save() if ( csa.learning_content.assignment_type == AssignmentType.PREP_ASSIGNMENT.value and start ): csa.submission_deadline.start = timezone.make_aware(start) csa.submission_deadline.end = None csa.submission_deadline.save() elif ( csa.learning_content.assignment_type == AssignmentType.CASEWORK.value and end ): csa.submission_deadline.start = timezone.make_aware( start ) + timezone.timedelta(days=30) csa.submission_deadline.end = None csa.submission_deadline.save() def create_or_update_course_session_edoniq_test( cs: CourseSession, course_slug: str, test_slug: str, start: datetime ): learning_content = LearningContentEdoniqTest.objects.filter( slug=f"{course_slug}-lp-circle-{test_slug}" ).first() if learning_content: cset, _created = CourseSessionEdoniqTest.objects.get_or_create( course_session=cs, learning_content=learning_content ) # trigger save to update due date cset.save() cset.deadline.start = timezone.make_aware(start) + timezone.timedelta(days=10) cset.deadline.end = None cset.deadline.save() def validate_row_data(data: Dict[str, any], required_headers: List[str]): logger.debug( "validate_row_data_missing_header", data={"data": data}, label="import", ) for header in required_headers: if str(data.get(header, "")).strip() in ["", "None"]: logger.debug( "validate_row_data_missing_header", data={"data": data, "header": header}, label="import", ) raise DataImportError(f"Missing or empty value for header {header}") def get_uk_course(language: str) -> Course: if language == "fr": course_id = COURSE_UK_FR elif language == "it": course_id = COURSE_UK_IT else: course_id = COURSE_UK return Course.objects.get(id=course_id) def import_trainers_from_excel_for_training( filename: str, language="de", course: Course = None ): workbook = load_workbook(filename=filename) sheet = workbook["Schulungen Trainer"] tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data( data, ["Email", "Vorname", "Name", "Sprache", "Klasse", "Generation"] ) create_or_update_trainer(course, data, language=language) def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"): course_title = course.title if course else "None" logger.debug( "create_or_update_trainer", course=course_title, data=data, label="import", ) user = create_or_update_user( email=data["Email"].lower(), first_name=data["Vorname"], last_name=data["Name"], ) user.language = data["Sprache"] user.save() group = data["Klasse"].strip() # general expert handling import_id = f"{data['Generation'].strip()} {group}" course_session = CourseSession.objects.filter( import_id=import_id, group=group, ).first() if course_session: csu, _created = CourseSessionUser.objects.get_or_create( course_session_id=course_session.id, user_id=user.id ) csu.role = CourseSessionUser.Role.EXPERT csu.save() else: logger.warning( "create_or_update_trainer: course_session not found", import_id=import_id, group=group, label="import", ) if not course and not course_session: logger.warning( "create_or_update_trainer: course_session and course are None", import_id=import_id, group=group, label="import", ) return if not course: course = course_session.course # circle expert handling circle_data = parse_circle_group_string(data["Circles"]) for circle_key in circle_data: circle_name = LP_DATA[circle_key][language]["title"] # print(circle_name, groups) import_id = f"{data['Generation'].strip()} {group}" course_session = CourseSession.objects.filter( import_id=import_id, group=group ).first() circle = Circle.objects.filter( slug=f"{course.slug}-lp-circle-{circle_name.lower()}" ).first() if course_session and circle: csu = CourseSessionUser.objects.filter( course_session_id=course_session.id, user_id=user.id ).first() if csu: csu.expert.add(circle) csu.save() def import_students_from_excel(filename: str): workbook = load_workbook(filename=filename) sheet = workbook.active tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) validate_row_data( data, [ "Email", "Vorname", "Name", "Sprache", "Durchführungen", "Lehrvertragsnummer", ], ) create_or_update_student(data) def create_or_update_student(data: Dict[str, Any]): logger.debug( "create_or_update_student", data=data, label="import", ) date_of_birth = _get_date_of_birth(data) user = create_or_update_user( email=data["Email"].lower(), first_name=data["Vorname"], last_name=data["Name"], contract_number=data.get("Lehrvertragsnummer", ""), date_of_birth=date_of_birth, ) user.language = data["Sprache"] update_user_json_data(user, data) user.save() # general expert handling import_id = data["Durchführungen"] course_session = CourseSession.objects.filter(import_id=import_id).first() if course_session: csu, _created = CourseSessionUser.objects.get_or_create( course_session_id=course_session.id, user_id=user.id ) csu.save() def _get_date_of_birth(data: Dict[str, Any]) -> str: date_of_birth = data.get("Geburtsdatum", None) if date_of_birth is None: return "" elif date_of_birth is date or date_of_birth is datetime: return date_of_birth.strftime("%d.%m.%Y") elif type(date_of_birth) is str: return date_of_birth def sync_students_from_t2l_excel(filename: str): workbook = load_workbook(filename=filename) sheet = workbook.active tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet) for row in tuple_list: data = dict(row) sync_students_from_t2l(data | {}) def sync_students_from_t2l(data): date_of_birth = _get_date_of_birth(data) try: user = User.objects.get( first_name=data["Vorname"], last_name=data["Name"], additional_json_data__Lehrvertragsnummer=data["Lehrvertragsnummer"], additional_json_data__Geburtsdatum=date_of_birth, ) except User.DoesNotExist: return # only sync data that is not in our user model for field in T2L_IGNORE_FIELDS: try: del data[field] except KeyError: pass update_user_json_data(user, data) user.save() def update_user_json_data(user: User, data: Dict[str, Any]): user.additional_json_data = user.additional_json_data | sanitize_json_data_input( data ) def sanitize_json_data_input(data: Dict[str, Any]) -> Dict[str, Any]: """ Saving additional_json_data fails if the data contains datetime objects. This is a quick and dirty fix to convert datetime objects to iso strings. """ for key, value in data.items(): if isinstance(value, datetime): data[key] = value.isoformat() elif isinstance(value, date): data[key] = value.isoformat() elif isinstance(value, time): data[key] = value.isoformat() else: data[key] = value return data