From ae62b436061b9f499d09c437fc11a55a4f26ccdb Mon Sep 17 00:00:00 2001 From: Christian Cueni Date: Tue, 10 Jan 2023 15:06:44 +0100 Subject: [PATCH] Fix upload, add delete s3 files command --- client/src/pages/learningPath/CirclePage.vue | 1 + server/config/settings/test.py | 10 +++++ server/vbv_lernwelt/core/admin.py | 11 ++++- server/vbv_lernwelt/core/managers.py | 19 +++++---- .../core/migrations/0004_user_sso_id.py | 24 +++++++++++ server/vbv_lernwelt/core/models.py | 4 +- .../course/tests/test_document_uploads.py | 20 ++++----- server/vbv_lernwelt/files/integrations.py | 3 +- .../commands/delete_unused_s3_files.py | 42 +++++++++++++++++++ server/vbv_lernwelt/sso/views.py | 13 +----- 10 files changed, 113 insertions(+), 34 deletions(-) create mode 100644 server/vbv_lernwelt/core/migrations/0004_user_sso_id.py create mode 100644 server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py diff --git a/client/src/pages/learningPath/CirclePage.vue b/client/src/pages/learningPath/CirclePage.vue index 8b30a6f4..c3ed4109 100644 --- a/client/src/pages/learningPath/CirclePage.vue +++ b/client/src/pages/learningPath/CirclePage.vue @@ -305,6 +305,7 @@ async function uploadDocument(data: DocumentUploadData) { @form-submit="uploadDocument" :learning-sequences="dropdownLearningSequences" :show-upload-error-message="showUploadErrorMessage" + :is-uploading="isUploading" /> diff --git a/server/config/settings/test.py b/server/config/settings/test.py index 5651c045..31a00571 100644 --- a/server/config/settings/test.py +++ b/server/config/settings/test.py @@ -14,6 +14,16 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend" +# Dummy data +AWS_S3_ACCESS_KEY_ID = "SOMEKEY" +AWS_S3_SECRET_ACCESS_KEY = "SOMEACCESSKEY" +AWS_STORAGE_BUCKET_NAME = "myvbv-dev.iterativ.ch" +AWS_S3_REGION_NAME = "eu-central-1" +AWS_S3_SIGNATURE_VERSION = "s3v4" +FILE_MAX_SIZE = 20971520 # 20MB +AWS_DEFAULT_ACL = "private" +AWS_PRESIGNED_EXPIRY = 300 + class DisableMigrations(dict): def __contains__(self, item): diff --git a/server/vbv_lernwelt/core/admin.py b/server/vbv_lernwelt/core/admin.py index 9e0be577..e22d7124 100644 --- a/server/vbv_lernwelt/core/admin.py +++ b/server/vbv_lernwelt/core/admin.py @@ -24,5 +24,12 @@ class UserAdmin(auth_admin.UserAdmin): ), (_("Important dates"), {"fields": ("last_login", "date_joined")}), ) - list_display = ["username", "first_name", "last_name", "is_active", "is_superuser"] - search_fields = ["first_name", "last_name", "email", "username"] + list_display = [ + "username", + "first_name", + "last_name", + "is_active", + "is_superuser", + "sso_id", + ] + search_fields = ["first_name", "last_name", "email", "username", "sso_id"] diff --git a/server/vbv_lernwelt/core/managers.py b/server/vbv_lernwelt/core/managers.py index 12389f45..eead0bf9 100644 --- a/server/vbv_lernwelt/core/managers.py +++ b/server/vbv_lernwelt/core/managers.py @@ -5,18 +5,21 @@ from django.contrib.auth.models import AbstractUser class UserManager(BaseUserManager): def create_or_update_by_email(self, user_dict: dict) -> tuple[AbstractUser, bool]: # create or sync user with OpenID Data - user, created = self.model.objects.get_or_create(sso_id=user_dict['oid'], defaults={ - "email": user_dict["email"], - "username": user_dict["email"], - "first_name": user_dict['first_name'], - "last_name": user_dict['last_name'], - }) + user, created = self.model.objects.get_or_create( + sso_id=user_dict["oid"], + defaults={ + "email": user_dict["email"], + "username": user_dict["email"], + "first_name": user_dict["first_name"], + "last_name": user_dict["last_name"], + }, + ) if not created: user.email = user_dict["email"] user.username = user_dict["email"] - user.first_name = user_dict['first_name'] - user.last_name = user_dict['last_name'] + user.first_name = user_dict["first_name"] + user.last_name = user_dict["last_name"] user.save() return user, created diff --git a/server/vbv_lernwelt/core/migrations/0004_user_sso_id.py b/server/vbv_lernwelt/core/migrations/0004_user_sso_id.py new file mode 100644 index 00000000..12201e7d --- /dev/null +++ b/server/vbv_lernwelt/core/migrations/0004_user_sso_id.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.13 on 2023-01-10 10:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0003_alter_user_managers"), + ] + + operations = [ + migrations.AddField( + model_name="user", + name="sso_id", + field=models.UUIDField( + blank=True, + default=None, + null=True, + unique=True, + verbose_name="SSO subscriber ID", + ), + ), + ] diff --git a/server/vbv_lernwelt/core/models.py b/server/vbv_lernwelt/core/models.py index feb0b69b..34f35c85 100644 --- a/server/vbv_lernwelt/core/models.py +++ b/server/vbv_lernwelt/core/models.py @@ -15,7 +15,9 @@ class User(AbstractUser): # objects = UserManager() avatar_url = models.CharField(max_length=254, blank=True, default="") email = models.EmailField("email address", unique=True) - sso_id = models.UUIDField('SSO subscriber ID', unique=True, null=True, blank=True, default=None) + sso_id = models.UUIDField( + "SSO subscriber ID", unique=True, null=True, blank=True, default=None + ) objects = UserManager() diff --git a/server/vbv_lernwelt/course/tests/test_document_uploads.py b/server/vbv_lernwelt/course/tests/test_document_uploads.py index 86066d7b..0f105647 100644 --- a/server/vbv_lernwelt/course/tests/test_document_uploads.py +++ b/server/vbv_lernwelt/course/tests/test_document_uploads.py @@ -1,4 +1,4 @@ -from django.conf import settings +from django.test import override_settings from rest_framework.test import APITestCase from vbv_lernwelt.core.create_default_users import create_default_users @@ -43,6 +43,7 @@ class DocumentUploadApiTestCase(APITestCase): username="patrizia.huggel@eiger-versicherungen.ch", password="myvbv1234" ) + @override_settings(FILE_UPLOAD_STORAGE="s3") def test_can_start_upload(self): ls = LearningSequence.objects.get( slug="test-lehrgang-lp-circle-analyse-ls-beobachten" @@ -54,16 +55,15 @@ class DocumentUploadApiTestCase(APITestCase): self.assertEqual(response.status_code, 200) self.assertNotEqual(response.data["url"], "") - if settings.FILE_UPLOAD_STORAGE == "s3": - self.assertTrue(response.data["url"].startswith("https://")) - self.assertEqual( - response.data["fields"]["Content-Type"], self.test_data["file_type"] - ) + self.assertTrue(response.data["url"].startswith("https://")) + self.assertEqual( + response.data["fields"]["Content-Type"], self.test_data["file_type"] + ) - self.assertEqual( - response.data["fields"]["Content-Disposition"], - f"attachment; filename={self.test_data['file_name']}", - ) + self.assertEqual( + response.data["fields"]["Content-Disposition"], + f"attachment; filename={self.test_data['file_name']}", + ) file_id = response.data["file_id"] file = UploadFile.objects.get(id=file_id) diff --git a/server/vbv_lernwelt/files/integrations.py b/server/vbv_lernwelt/files/integrations.py index 11585466..7e7b5d5f 100644 --- a/server/vbv_lernwelt/files/integrations.py +++ b/server/vbv_lernwelt/files/integrations.py @@ -119,8 +119,7 @@ def s3_delete_file(*, file_path: str): credentials = s3_get_credentials() s3_client = s3_get_client() - some = s3_client.delete_object( + s3_client.delete_object( Bucket=credentials.bucket_name, Key=file_path, ) - pass diff --git a/server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py b/server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py new file mode 100644 index 00000000..b776f354 --- /dev/null +++ b/server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py @@ -0,0 +1,42 @@ +from django.core.management.base import BaseCommand + +from vbv_lernwelt.files.integrations import s3_get_client, s3_get_credentials +from vbv_lernwelt.files.models import UploadFile + + +class Command(BaseCommand): + help = "Delete unused files from S3" + + def add_arguments(self, parser): + parser.add_argument( + "--dry-run", + action="store_true", + dest="dry_run", + default=False, + help="Dry run", + ) + + def handle(self, *args, **options): + dry_run = options["dry_run"] + credentials = s3_get_credentials() + s3_client = s3_get_client() + num_deleted = 0 + + if dry_run: + print("------ DRY RUN -------") + + result = s3_client.list_objects_v2( + Bucket=credentials.bucket_name, Prefix="circledocuments/" + ) + for contents in result.get("Contents"): + try: + UploadFile.objects.get(file=contents["Key"]) + except UploadFile.DoesNotExist: + print(f"Deleting {contents['Key']}") + if not dry_run: + s3_client.delete_object( + Bucket=credentials.bucket_name, Key=contents["Key"] + ) + num_deleted += 1 + + print(f"Deleted {num_deleted} files in bucket {credentials.bucket_name}") diff --git a/server/vbv_lernwelt/sso/views.py b/server/vbv_lernwelt/sso/views.py index e068f7a7..7bbed403 100644 --- a/server/vbv_lernwelt/sso/views.py +++ b/server/vbv_lernwelt/sso/views.py @@ -1,10 +1,7 @@ -import json - import structlog as structlog from authlib.integrations.base_client import OAuthError from django.conf import settings from django.contrib.auth import get_user_model, login as dj_login -from django.http import HttpResponse from django.shortcuts import redirect from sentry_sdk import capture_exception @@ -30,7 +27,6 @@ def authorize(request): request ) deocded_token = decode_jwt(token["id_token"]) - return HttpResponse(json.dumps(deocded_token)) except OAuthError as e: logger.error(f"OAuth error: {e}") if not settings.DEBUG: @@ -38,12 +34,7 @@ def authorize(request): return redirect(f"/{OAUTH_FAIL_REDIRECT}?state=someerror") # to be defined user_data = _user_data_from_token_data(deocded_token) - user, created = get_user_model().objects.create_or_update_by_email( - user_data["email"], - user_data["first_name"], - user_data["last_name"], - user_data["username"], - ) + user, created = get_user_model().objects.create_or_update_by_email(user_data) dj_login(request, user) # todo: redirect to other page if new user @@ -57,5 +48,5 @@ def _user_data_from_token_data(token: dict) -> dict: "last_name": token.get("family_name", ""), "username": token.get("preferred_username", first_email), "email": first_email, - "sub": token.get("sub") + "oid": token.get("oid"), }