Fix upload, add delete s3 files command

client/src/pages/learningPath/CirclePage.vue

@@ -305,6 +305,7 @@ async function uploadDocument(data: DocumentUploadData) {
               @form-submit="uploadDocument"
               :learning-sequences="dropdownLearningSequences"
               :show-upload-error-message="showUploadErrorMessage"
+              :is-uploading="isUploading"
             />
           </template>
         </ItModal>

server/config/settings/test.py

@@ -14,6 +14,16 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
 
+# Dummy data
+AWS_S3_ACCESS_KEY_ID = "SOMEKEY"
+AWS_S3_SECRET_ACCESS_KEY = "SOMEACCESSKEY"
+AWS_STORAGE_BUCKET_NAME = "myvbv-dev.iterativ.ch"
+AWS_S3_REGION_NAME = "eu-central-1"
+AWS_S3_SIGNATURE_VERSION = "s3v4"
+FILE_MAX_SIZE = 20971520  # 20MB
+AWS_DEFAULT_ACL = "private"
+AWS_PRESIGNED_EXPIRY = 300
+
 
 class DisableMigrations(dict):
     def __contains__(self, item):

server/vbv_lernwelt/core/admin.py

@@ -24,5 +24,12 @@ class UserAdmin(auth_admin.UserAdmin):
         ),
         (_("Important dates"), {"fields": ("last_login", "date_joined")}),
     )
-    list_display = ["username", "first_name", "last_name", "is_active", "is_superuser"]
+    list_display = [
-    search_fields = ["first_name", "last_name", "email", "username"]
+        "username",
+        "first_name",
+        "last_name",
+        "is_active",
+        "is_superuser",
+        "sso_id",
+    ]
+    search_fields = ["first_name", "last_name", "email", "username", "sso_id"]

server/vbv_lernwelt/core/managers.py

@@ -5,18 +5,21 @@ from django.contrib.auth.models import AbstractUser
 class UserManager(BaseUserManager):
     def create_or_update_by_email(self, user_dict: dict) -> tuple[AbstractUser, bool]:
         # create or sync user with OpenID Data
-        user, created = self.model.objects.get_or_create(sso_id=user_dict['oid'], defaults={
+        user, created = self.model.objects.get_or_create(
-            "email": user_dict["email"],
+            sso_id=user_dict["oid"],
-            "username": user_dict["email"],
+            defaults={
-            "first_name": user_dict['first_name'],
+                "email": user_dict["email"],
-            "last_name": user_dict['last_name'],
+                "username": user_dict["email"],
-        })
+                "first_name": user_dict["first_name"],
+                "last_name": user_dict["last_name"],
+            },
+        )
 
         if not created:
             user.email = user_dict["email"]
             user.username = user_dict["email"]
-            user.first_name = user_dict['first_name']
+            user.first_name = user_dict["first_name"]
-            user.last_name = user_dict['last_name']
+            user.last_name = user_dict["last_name"]
             user.save()
 
         return user, created

server/vbv_lernwelt/core/migrations/0004_user_sso_id.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.2.13 on 2023-01-10 10:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0003_alter_user_managers"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="user",
+            name="sso_id",
+            field=models.UUIDField(
+                blank=True,
+                default=None,
+                null=True,
+                unique=True,
+                verbose_name="SSO subscriber ID",
+            ),
+        ),
+    ]

server/vbv_lernwelt/core/models.py

@@ -15,7 +15,9 @@ class User(AbstractUser):
     # objects = UserManager()
     avatar_url = models.CharField(max_length=254, blank=True, default="")
     email = models.EmailField("email address", unique=True)
-    sso_id = models.UUIDField('SSO subscriber ID', unique=True, null=True, blank=True, default=None)
+    sso_id = models.UUIDField(
+        "SSO subscriber ID", unique=True, null=True, blank=True, default=None
+    )
 
     objects = UserManager()
 

server/vbv_lernwelt/course/tests/test_document_uploads.py

@@ -1,4 +1,4 @@
-from django.conf import settings
+from django.test import override_settings
 from rest_framework.test import APITestCase
 
 from vbv_lernwelt.core.create_default_users import create_default_users
@@ -43,6 +43,7 @@ class DocumentUploadApiTestCase(APITestCase):
             username="patrizia.huggel@eiger-versicherungen.ch", password="myvbv1234"
         )
 
+    @override_settings(FILE_UPLOAD_STORAGE="s3")
     def test_can_start_upload(self):
         ls = LearningSequence.objects.get(
             slug="test-lehrgang-lp-circle-analyse-ls-beobachten"
@@ -54,16 +55,15 @@ class DocumentUploadApiTestCase(APITestCase):
         self.assertEqual(response.status_code, 200)
         self.assertNotEqual(response.data["url"], "")
 
-        if settings.FILE_UPLOAD_STORAGE == "s3":
+        self.assertTrue(response.data["url"].startswith("https://"))
-            self.assertTrue(response.data["url"].startswith("https://"))
+        self.assertEqual(
-            self.assertEqual(
+            response.data["fields"]["Content-Type"], self.test_data["file_type"]
-                response.data["fields"]["Content-Type"], self.test_data["file_type"]
+        )
-            )
 
-            self.assertEqual(
+        self.assertEqual(
-                response.data["fields"]["Content-Disposition"],
+            response.data["fields"]["Content-Disposition"],
-                f"attachment; filename={self.test_data['file_name']}",
+            f"attachment; filename={self.test_data['file_name']}",
-            )
+        )
 
         file_id = response.data["file_id"]
         file = UploadFile.objects.get(id=file_id)

server/vbv_lernwelt/files/integrations.py

@@ -119,8 +119,7 @@ def s3_delete_file(*, file_path: str):
     credentials = s3_get_credentials()
     s3_client = s3_get_client()
 
-    some = s3_client.delete_object(
+    s3_client.delete_object(
         Bucket=credentials.bucket_name,
         Key=file_path,
     )
-    pass

server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py

@@ -0,0 +1,42 @@
+from django.core.management.base import BaseCommand
+
+from vbv_lernwelt.files.integrations import s3_get_client, s3_get_credentials
+from vbv_lernwelt.files.models import UploadFile
+
+
+class Command(BaseCommand):
+    help = "Delete unused files from S3"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            dest="dry_run",
+            default=False,
+            help="Dry run",
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options["dry_run"]
+        credentials = s3_get_credentials()
+        s3_client = s3_get_client()
+        num_deleted = 0
+
+        if dry_run:
+            print("------ DRY RUN -------")
+
+        result = s3_client.list_objects_v2(
+            Bucket=credentials.bucket_name, Prefix="circledocuments/"
+        )
+        for contents in result.get("Contents"):
+            try:
+                UploadFile.objects.get(file=contents["Key"])
+            except UploadFile.DoesNotExist:
+                print(f"Deleting {contents['Key']}")
+                if not dry_run:
+                    s3_client.delete_object(
+                        Bucket=credentials.bucket_name, Key=contents["Key"]
+                    )
+                num_deleted += 1
+
+        print(f"Deleted {num_deleted} files in bucket {credentials.bucket_name}")

server/vbv_lernwelt/sso/views.py

@@ -1,10 +1,7 @@
-import json
-
 import structlog as structlog
 from authlib.integrations.base_client import OAuthError
 from django.conf import settings
 from django.contrib.auth import get_user_model, login as dj_login
-from django.http import HttpResponse
 from django.shortcuts import redirect
 from sentry_sdk import capture_exception
 
@@ -30,7 +27,6 @@ def authorize(request):
             request
         )
         deocded_token = decode_jwt(token["id_token"])
-        return HttpResponse(json.dumps(deocded_token))
     except OAuthError as e:
         logger.error(f"OAuth error: {e}")
         if not settings.DEBUG:
@@ -38,12 +34,7 @@ def authorize(request):
         return redirect(f"/{OAUTH_FAIL_REDIRECT}?state=someerror")  # to be defined
 
     user_data = _user_data_from_token_data(deocded_token)
-    user, created = get_user_model().objects.create_or_update_by_email(
+    user, created = get_user_model().objects.create_or_update_by_email(user_data)
-        user_data["email"],
-        user_data["first_name"],
-        user_data["last_name"],
-        user_data["username"],
-    )
 
     dj_login(request, user)
     # todo: redirect to other page if new user
@@ -57,5 +48,5 @@ def _user_data_from_token_data(token: dict) -> dict:
         "last_name": token.get("family_name", ""),
         "username": token.get("preferred_username", first_email),
         "email": first_email,
-        "sub": token.get("sub")
+        "oid": token.get("oid"),
     }