ramon
/
vbv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add real courses data, update importer 

Make course optional on import

Add importer in admin

Use new trainer format in xls

Import trainers via admin interface

Add participant import

Update tests

Update url, handle error

Refactor importer

Add json field

Fix tests

Add update test
This commit is contained in:
Christian Cueni 2023-07-13 08:38:36 +02:00
parent 65d527d894
commit 12977b01cc
13 changed files with 378 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/config/settings/base.py
View File

@ -122,6 +122,7 @@ LOCAL_APPS = [
"vbv_lernwelt.notify",
"vbv_lernwelt.assignment",
"vbv_lernwelt.duedate",
"vbv_lernwelt.importer",
]
# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

16
server/config/urls.py
View File

@ -42,6 +42,10 @@ from vbv_lernwelt.feedback.views import (
get_expert_feedbacks_for_course,
get_feedback_for_circle,
)
from vbv_lernwelt.importer.views import (
coursesessions_students_import,
coursesessions_trainers_import,
)
from vbv_lernwelt.notify.views import email_notification_settings
from wagtail import urls as wagtail_urls
from wagtail.admin import urls as wagtailadmin_urls
@ -143,6 +147,18 @@ urlpatterns = [
# edoniq test
path(r'api/core/edoniq-test/export-users/', export_students, name='edoniq_export_students'),
# importer
path(
r"server/importer/coursesession-trainer-import/",
coursesessions_trainers_import,
name="coursesessions_trainers_import",
),
path(
r"server/importer/coursesession-students-import/",
coursesessions_students_import,
name="coursesessions_students_import",
),
# testing and debug
path('server/raise_error/',
user_passes_test(lambda u: u.is_superuser, login_url='/login/')(

1
server/vbv_lernwelt/core/admin.py
View File

@ -23,6 +23,7 @@ class UserAdmin(auth_admin.UserAdmin):
},
),
(_("Important dates"), {"fields": ("last_login", "date_joined")}),
(_("Additional data"), {"fields": ("additional_json_data",)}),
)
list_display = [
"username",

25
server/vbv_lernwelt/course/management/commands/create_default_courses.py
View File

@ -79,7 +79,7 @@ from vbv_lernwelt.feedback.creators.create_demo_feedback import create_feedback
from vbv_lernwelt.importer.services import (
import_course_sessions_from_excel,
import_students_from_excel,
import_trainers_from_excel,
import_trainers_from_excel_for_training,
)
from vbv_lernwelt.learnpath.create_vv_new_learning_path import (
create_vv_new_learning_path,
@ -547,12 +547,13 @@ def create_course_training_de():
print(current_dir)
course = Course.objects.get(id=COURSE_UK_TRAINING)
import_course_sessions_from_excel(
course,
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
course=course,
restrict_language="de",
)
import_trainers_from_excel(
course,
import_trainers_from_excel_for_training(
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
course=course,
)
import_students_from_excel(
f"{current_dir}/../../../importer/tests/Schulungen_Teilnehmende.xlsx",
@ -636,13 +637,13 @@ def create_course_training_fr():
print(current_dir)
course = Course.objects.get(id=COURSE_UK_TRAINING_FR)
import_course_sessions_from_excel(
course,
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
language="fr",
restrict_language="fr",
course=course,
)
import_trainers_from_excel(
course,
import_trainers_from_excel_for_training(
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
course=course,
language="fr",
)
import_students_from_excel(
@ -723,13 +724,13 @@ def create_course_training_it():
print(current_dir)
course = Course.objects.get(id=COURSE_UK_TRAINING_IT)
import_course_sessions_from_excel(
course,
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
language="it",
restrict_language="it",
course=course,
)
import_trainers_from_excel(
course,
import_trainers_from_excel_for_training(
f"{current_dir}/../../../importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx",
course=course,
language="it",
)
import_students_from_excel(

175
server/vbv_lernwelt/importer/services.py
View File

@ -1,9 +1,10 @@
from typing import Any, Dict
from typing import Any, Dict, List
import structlog
from openpyxl.reader.excel import load_workbook
from vbv_lernwelt.core.models import User
from vbv_lernwelt.course.consts import COURSE_UK, COURSE_UK_FR, COURSE_UK_IT
from vbv_lernwelt.course.models import Course, CourseSession, CourseSessionUser
from vbv_lernwelt.course_session.models import CourseSessionAttendanceCourse
from vbv_lernwelt.importer.utils import (
@ -15,9 +16,25 @@ from vbv_lernwelt.learnpath.models import Circle, LearningContentAttendanceCours
logger = structlog.get_logger(__name__)
CIRCLE_NAMES = {
"Kickoff": {"de": "Kickoff", "fr": "Lancement", "it": "Introduzione"},
"Basis": {"de": "Basis", "fr": "Base", "it": "Base"},
"Fahrzeug": {"de": "Fahrzeug", "fr": "Véhicule", "it": "Veicolo"},
"Haushalt Teil 1": {"de": "Haushalt Teil 1", "fr": "Habitat partie 1", "it": ""},
"Haushalt Teil 2": {"de": "Haushalt Teil 2", "fr": "Haushalt Teil 2", "it": ""},
}
class DataImportError(Exception):
pass
def create_or_update_user(
email: str, first_name: str = "", last_name: str = "", sso_id: str = None
email: str,
first_name: str = "",
last_name: str = "",
sso_id: str = None,
contract_number: str = "",
):
logger.debug(
"create_or_update_user",
@ -33,6 +50,13 @@ def create_or_update_user(
if user_qs.exists():
user = user_qs.first()
if not user and contract_number:
user_qs = User.objects.filter(
additional_json_data__Lehrvertragsnummer=contract_number
)
if user_qs.exists():
user = user_qs.first()
if not user:
user_qs = User.objects.filter(email=email)
if user_qs.exists():
@ -53,25 +77,41 @@ def create_or_update_user(
return user
def import_course_sessions_from_excel(course: Course, filename: str, language="de"):
def import_course_sessions_from_excel(
filename: str, course: Course = None, restrict_language=None
):
workbook = load_workbook(filename=filename)
sheet = workbook["Schulungen Durchführung"]
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
data = dict(row)
validate_row_data(data, ["Klasse", "ID", "Generation", "Region", "Sprache"])
language = data["Sprache"].strip()
# this can be removed when the training import (create_course_training_xx) is no longer used
if restrict_language and language != restrict_language:
continue
if not course:
course = get_uk_course(language)
create_or_update_course_session(
course, dict(row), language=language, circles=["Fahrzeug"]
course, data, language, circle_keys=["Kickoff", "Basis", "Fahrzeug"]
)
def create_or_update_course_session(
course: Course, data: Dict[str, Any], language="de", circles=None
course: Course,
data: Dict[str, Any],
language: str,
circle_keys=None,
):
"""
:param data: the following keys are required to process the data: Generation, Region, Klasse
:return:
"""
logger.debug(
"create_or_update_course_session",
course=course.title,
@ -79,10 +119,9 @@ def create_or_update_course_session(
label="import",
)
if circles is None:
circles = []
if not circle_keys:
circle_keys = []
# TODO: validation
group = data["Klasse"].strip()
import_id = data["ID"].strip()
@ -91,12 +130,8 @@ def create_or_update_course_session(
title = f"{region} {generation} {group}"
if not import_id.lower().startswith(language.lower()):
# FIXME: language check depends on import_id format for now...
return None
cs, _created = CourseSession.objects.get_or_create(
import_id=import_id, group=group, course=course
title=title, course=course, import_id=import_id
)
cs.additional_json_data["import_data"] = data
@ -109,29 +144,26 @@ def create_or_update_course_session(
cs.import_id = import_id
cs.save()
for circle in circles:
attendance_course_lp_qs = None
if language == "de":
attendance_course_lp_qs = LearningContentAttendanceCourse.objects.filter(
slug=f"{course.slug}-lp-circle-{circle.lower()}-lc-präsenzkurs-{circle.lower()}"
)
for circle in circle_keys:
circle_name = CIRCLE_NAMES[circle][language]
elif language == "fr":
# todo: this is a hack remove me
attendance_course_lp_qs = LearningContentAttendanceCourse.objects.filter(
slug=f"{course.slug}-lp-circle-véhicule-lc-cours-de-présence-véhicule-à-moteur"
)
elif language == "it":
# todo: this is a hack remove me
attendance_course_lp_qs = LearningContentAttendanceCourse.objects.filter(
slug=f"{course.slug}-lp-circle-veicolo-lc-corso-di-presenza-veicolo"
slug=f"{course.slug}-lp-circle-{circle_name.lower()}-lc-präsenzkurs-{circle_name.lower()}"
)
if attendance_course_lp_qs and attendance_course_lp_qs.exists():
# reset existing data
# TODO: Is this save? stuff shouldn't get lost
CourseSessionAttendanceCourse.objects.filter(
course_session=cs, learning_content=attendance_course_lp_qs.first()
).delete()
location = f"{data[f'{circle} Raum']}, {data[f'{circle} Standort']}, {data[f'{circle} Adresse']}"
csa = CourseSessionAttendanceCourse.objects.create(
course_session=cs,
learning_content=attendance_course_lp_qs.first(),
location=data[f"{circle} Raum"],
location=location,
trainer="",
)
csa.due_date.start = try_parse_datetime(data[f"{circle} Start"])[1]
@ -141,19 +173,50 @@ def create_or_update_course_session(
return cs
def import_trainers_from_excel(course: Course, filename: str, language="de"):
def validate_row_data(data: Dict[str, any], required_headers: List[str]):
for header in required_headers:
some = str(data.get(header, "")).strip()
if str(data.get(header, "")).strip() in ["", "None"]:
logger.debug(
"validate_row_data_missing_header",
data={"data": data, "header": header},
label="import",
)
raise DataImportError(f"Missing or empty value for header {header}")
def get_uk_course(language: str) -> Course:
if language == "fr":
course_id = COURSE_UK_FR
elif language == "it":
course_id = COURSE_UK_IT
else:
course_id = COURSE_UK
return Course.objects.get(id=course_id)
def import_trainers_from_excel_for_training(
filename: str, language="de", course: Course = None
):
workbook = load_workbook(filename=filename)
sheet = workbook["Schulungen Trainer"]
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
create_or_update_trainer(course, dict(row), language=language)
data = dict(row)
validate_row_data(
data, ["Email", "Vorname", "Name", "Sprache", "Klasse", "Generation"]
)
create_or_update_trainer(course, data, language=language)
def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"):
course_title = course.title if course else "None"
logger.debug(
"create_or_update_trainer",
course=course.title,
course=course_title,
data=data,
label="import",
)
@ -163,11 +226,12 @@ def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"
first_name=data["Vorname"],
last_name=data["Name"],
)
user.language = data["Sprache"]
user.save()
groups = [g.strip() for g in data["Klasse"].strip().split(",")]
group = data["Klasse"].strip()
# general expert handling
for group in groups:
import_id = f"{data['Generation'].strip()} {group}"
course_session = CourseSession.objects.filter(
import_id=import_id,
@ -187,23 +251,25 @@ def create_or_update_trainer(course: Course, data: Dict[str, Any], language="de"
label="import",
)
if not course:
if not course and not course_session:
logger.warning(
"create_or_update_trainer: course_session and course are None",
import_id=import_id,
group=group,
label="import",
)
return
if not course:
course = course_session.course
# circle expert handling
circle_data = parse_circle_group_string(data["Circles"])
for circle_string in circle_data:
parts = circle_string.split("(", 1)
circle_name = parts[0].strip()
for circle_key in circle_data:
groups = [g.strip() for g in parts[1].rstrip(")").strip().split(",")]
# FIXME: hardcoded translation
if language == "fr" and circle_name == "Fahrzeug":
circle_name = "Véhicule"
circle_name = CIRCLE_NAMES[circle_key][language]
# print(circle_name, groups)
for group in groups:
import_id = f"{data['Generation'].strip()} {group}"
course_session = CourseSession.objects.filter(
import_id=import_id, group=group
@ -227,7 +293,18 @@ def import_students_from_excel(filename: str):
tuple_list = calc_header_tuple_list_from_pyxl_sheet(sheet)
for row in tuple_list:
create_or_update_student(dict(row))
data = dict(row)
validate_row_data(
data,
[
"Email",
"Vorname",
"Name",
"Sprache",
"Durchführungen",
],
)
create_or_update_student(data)
def create_or_update_student(data: Dict[str, Any]):
@ -241,13 +318,15 @@ def create_or_update_student(data: Dict[str, Any]):
email=data["Email"].lower(),
first_name=data["Vorname"],
last_name=data["Name"],
contract_number=data.get("Lehrvertragsnummer", ""),
)
# TODO: handle language
user.language = data["Sprache"]
user.additional_json_data = user.additional_json_data | data
user.save()
# general expert handling
import_ids = [i.strip() for i in data["Durchführungen"].split(",")]
for import_id in import_ids:
import_id = data["Durchführungen"]
course_session = CourseSession.objects.filter(import_id=import_id).first()
if course_session:
csu, _created = CourseSessionUser.objects.get_or_create(

12
server/vbv_lernwelt/importer/templates/admin/importer/error.html Normal file
View File

@ -0,0 +1,12 @@
{% extends "admin/index.html" %}
{% block content %}
<div id="content-main">
<div class="content">
<h1>Die Daten konnten nicht importiert werden</h1>
<p>Folgender Fehler ist aufgetreten:</p>
<pre>{{ error }}</pre>
</div>
</div>
{% endblock %}

BIN
server/vbv_lernwelt/importer/tests/Schulungen_Durchfuehrung_Trainer.xlsx
View File

Binary file not shown.

BIN
server/vbv_lernwelt/importer/tests/Schulungen_Teilnehmende.xlsx
View File

Binary file not shown.

49
server/vbv_lernwelt/importer/tests/test_import_course_sessions.py
View File

@ -6,7 +6,11 @@ from openpyxl.reader.excel import load_workbook
from vbv_lernwelt.course.creators.test_course import create_test_course
from vbv_lernwelt.course.models import CourseSession
from vbv_lernwelt.course_session.models import CourseSessionAttendanceCourse
from vbv_lernwelt.importer.services import create_or_update_course_session
from vbv_lernwelt.importer.services import (
create_or_update_course_session,
DataImportError,
validate_row_data,
)
from vbv_lernwelt.importer.utils import calc_header_tuple_list_from_pyxl_sheet
test_dir = os.path.dirname(os.path.abspath(__file__))
@ -26,10 +30,10 @@ class ImportCourseSessionTestCase(TestCase):
for row in tuple_list:
print(row)
create_or_update_course_session(
self.course, dict(row), language="de", circles=["Fahrzeug"]
self.course, dict(row), language="de", circle_keys=["Fahrzeug"]
)
self.assertEqual(CourseSession.objects.count(), 3)
self.assertEqual(CourseSession.objects.count(), 6)
class CreateOrUpdateCourseSessionTestCase(TestCase):
@ -41,6 +45,7 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
("ID", "DE 2023 A"),
("Generation", 2023),
("Region", "Deutschschweiz"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 13:30"),
("Fahrzeug Ende", "06.06.2023, 15:00"),
@ -54,7 +59,9 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
data = dict(row)
cs = create_or_update_course_session(self.course, data, circles=["Fahrzeug"])
cs = create_or_update_course_session(
self.course, data, language="de", circle_keys=["Fahrzeug"]
)
self.assertEqual(cs.import_id, "DE 2023 A")
self.assertEqual(cs.title, "Deutschschweiz 2023 A")
@ -82,6 +89,7 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
("ID", "DE 2023"),
("Generation", 2023),
("Region", "Deutschschweiz"),
("Sprache", "de"),
("Klasse", "A"),
("Fahrzeug Start", "06.06.2023, 13:30"),
("Fahrzeug Ende", "06.06.2023, 15:00"),
@ -95,7 +103,9 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
data = dict(row)
cs = create_or_update_course_session(self.course, data, circles=["Fahrzeug"])
cs = create_or_update_course_session(
self.course, data, language="de", circle_keys=["Fahrzeug"]
)
self.assertEqual(1, CourseSession.objects.count())
@ -112,3 +122,32 @@ class CreateOrUpdateCourseSessionTestCase(TestCase):
self.assertEqual(
attendance_course.due_date.end.isoformat(), "2023-06-06T13:00:00+00:00"
)
def test_raise_exception_if_header_is_missing(self):
data = [
("ID", "DE 2023"),
]
with self.assertRaises(DataImportError):
validate_row_data(
dict(data),
[
"ID",
"Generation",
],
)
def test_raise_exception_if_required_field_is_empty(self):
data = [
("ID", "DE 2023"),
("Generation", ""),
]
with self.assertRaises(DataImportError):
validate_row_data(
dict(data),
[
"ID",
"Generation",
],
)

62
server/vbv_lernwelt/importer/tests/test_import_students.py
View File

@ -1,5 +1,4 @@
import os
from datetime import datetime
from django.test import TestCase
from openpyxl.reader.excel import load_workbook
@ -44,33 +43,56 @@ class CreateOrUpdateStudentTestCase(TestCase):
group="A",
)
def test_create_student(self):
row = [
("Name", "Rascher"),
("Vorname", "Barbara"),
("Email", "barbara.rascher@vbv-afa.ch"),
("Sprache", "de"),
("Durchführungen", "DE 2023 A"),
("Datum", datetime(2023, 9, 6, 0, 0)),
(None, "VBV"),
(None, None),
(None, None),
(None, None),
(None, None),
]
self.user_dict = {
"Name": "Rascher",
"Vorname": "Barbara",
"Email": "barbara.rascher@vbv-afa.ch",
"Sprache": "de",
"Durchführungen": "DE 2023 A",
"Lehrvertragsnummer": "1234",
"Tel. Privat": "079 593 83 43",
}
create_or_update_student(dict(row))
def test_create_student(self):
create_or_update_student(self.user_dict)
self.assertEqual(
CourseSessionUser.objects.filter(
user__email=self.user_dict["Email"]
).count(),
1,
)
csu = CourseSessionUser.objects.filter(
course_session=self.course_session_a,
).first()
self.assertEqual(csu.role, CourseSessionUser.Role.MEMBER)
self.assertEqual(csu.user.email, self.user_dict["Email"])
self.assertEqual(csu.user.additional_json_data, self.user_dict)
def test_update_student(self):
create_or_update_student(self.user_dict)
self.user_dict["Email"] = "br@vbv.ch"
create_or_update_student(self.user_dict)
self.assertEqual(
CourseSessionUser.objects.filter(
user__email="barbara.rascher@vbv-afa.ch"
).count(),
0,
)
self.assertEqual(
CourseSessionUser.objects.filter(
user__additional_json_data__Lehrvertragsnummer=self.user_dict["Lehrvertragsnummer"]
).count(),
1,
)
csu = CourseSessionUser.objects.get(
csu = CourseSessionUser.objects.filter(
course_session=self.course_session_a,
)
).first()
self.assertEqual(csu.role, CourseSessionUser.Role.MEMBER)
self.assertEqual(csu.user.email, "barbara.rascher@vbv-afa.ch")
self.assertEqual(csu.user.additional_json_data, self.user_dict)

20
server/vbv_lernwelt/importer/tests/test_import_trainers.py
View File

@ -58,18 +58,28 @@ class CreateOrUpdateTrainerTestCase(TestCase):
)
def test_create_trainer(self):
row = [
rows = [
[
("Name", "Hänni"),
("Vorname", "Fabienne"),
("Email", "fabienne.haenni@vbv-afa.ch"),
("Sprache", "de"),
("Generation", "DE 2023"),
("Klasse", "A, B"),
("Circles", "Fahrzeug (A, B), Reisen (A), KMU (B)"),
("Status Referenten", "ok"),
(None, "Schulung D"),
("Klasse", "A"),
("Circles", "Fahrzeug, Haushalt Teil 1"),
],
[
("Name", "Hänni"),
("Vorname", "Fabienne"),
("Email", "fabienne.haenni@vbv-afa.ch"),
("Sprache", "de"),
("Generation", "DE 2023"),
("Klasse", "B"),
("Circles", "Fahrzeug, Haushalt Teil 2"),
],
]
for row in rows:
create_or_update_trainer(self.course, dict(row))
self.assertEqual(

48
server/vbv_lernwelt/importer/views.py
View File

@ -0,0 +1,48 @@
from typing import Callable
from django.contrib import messages
from django.contrib.admin.views.decorators import staff_member_required
from django.shortcuts import redirect, render
from vbv_lernwelt.importer.services import (
import_course_sessions_from_excel,
import_students_from_excel,
import_trainers_from_excel_for_training,
)
@staff_member_required
def coursesessions_trainers_import(request):
return handle_import(
request,
"Die Durchführungen und Trainer wurden erfolgreich importiert!",
import_training_and_trainer,
)
def import_training_and_trainer(excel_file: str):
import_course_sessions_from_excel(
excel_file,
)
import_trainers_from_excel_for_training(excel_file)
@staff_member_required
def coursesessions_students_import(request):
return handle_import(
request,
"Die Teilnehmer wurden erflogreich importiert!",
import_students_from_excel,
)
def handle_import(request, success_msg: str, importer: Callable[[str], None]):
if request.method == "POST" and request.FILES["excel_file"]:
excel_file = request.FILES["excel_file"]
try:
importer(excel_file)
except Exception as e:
return render(request, "admin/importer/error.html", {"error": str(e)})
messages.info(request, success_msg)
return redirect("admin:index")

15
server/vbv_lernwelt/templates/admin/index.html
View File

@ -6,6 +6,21 @@
<div class="content">
<h1>Export üK</h1>
<h1>Excel Import üK</h1>
<h2>Durchführungen und Trainer</h2>
<form method="post" enctype="multipart/form-data" action="/server/importer/coursesession-trainer-import/">
{% csrf_token %}
<input type="file" name="excel_file" accept=".xlsx, .xls">
<input type="submit" value="Durchführungen und Trainer importieren">
</form>
<h2>Teilnehmer</h2>
<form method="post" enctype="multipart/form-data" action="/server/importer/coursesession-students-import/">
{% csrf_token %}
<input type="file" name="excel_file" accept=".xlsx, .xls">
<input type="submit" value="Teilnehmer importieren">
</form>
<h2>Edoniq Teilnehmer</h2>
<a href="{% url 'edoniq_export_students' %}" class="btn btn-primary">Teilnehmer exportieren</a>