ramon
/
vbv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix upload, add delete s3 files command

This commit is contained in:
Christian Cueni 2023-01-10 15:06:44 +01:00
parent e2ce704a40
commit ae62b43606
10 changed files with 113 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/src/pages/learningPath/CirclePage.vue
View File

@ -305,6 +305,7 @@ async function uploadDocument(data: DocumentUploadData) {
@form-submit="uploadDocument"
:learning-sequences="dropdownLearningSequences"
:show-upload-error-message="showUploadErrorMessage"
:is-uploading="isUploading"
/>
</template>
</ItModal>

10
server/config/settings/test.py
View File

@ -14,6 +14,16 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
# https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
# Dummy data
AWS_S3_ACCESS_KEY_ID = "SOMEKEY"
AWS_S3_SECRET_ACCESS_KEY = "SOMEACCESSKEY"
AWS_STORAGE_BUCKET_NAME = "myvbv-dev.iterativ.ch"
AWS_S3_REGION_NAME = "eu-central-1"
AWS_S3_SIGNATURE_VERSION = "s3v4"
FILE_MAX_SIZE = 20971520 # 20MB
AWS_DEFAULT_ACL = "private"
AWS_PRESIGNED_EXPIRY = 300
class DisableMigrations(dict):
def __contains__(self, item):

11
server/vbv_lernwelt/core/admin.py
View File

@ -24,5 +24,12 @@ class UserAdmin(auth_admin.UserAdmin):
),
(_("Important dates"), {"fields": ("last_login", "date_joined")}),
)
list_display = ["username", "first_name", "last_name", "is_active", "is_superuser"]
search_fields = ["first_name", "last_name", "email", "username"]
list_display = [
"username",
"first_name",
"last_name",
"is_active",
"is_superuser",
"sso_id",
]
search_fields = ["first_name", "last_name", "email", "username", "sso_id"]

19
server/vbv_lernwelt/core/managers.py
View File

@ -5,18 +5,21 @@ from django.contrib.auth.models import AbstractUser
class UserManager(BaseUserManager):
def create_or_update_by_email(self, user_dict: dict) -> tuple[AbstractUser, bool]:
# create or sync user with OpenID Data
user, created = self.model.objects.get_or_create(sso_id=user_dict['oid'], defaults={
"email": user_dict["email"],
"username": user_dict["email"],
"first_name": user_dict['first_name'],
"last_name": user_dict['last_name'],
})
user, created = self.model.objects.get_or_create(
sso_id=user_dict["oid"],
defaults={
"email": user_dict["email"],
"username": user_dict["email"],
"first_name": user_dict["first_name"],
"last_name": user_dict["last_name"],
},
)
if not created:
user.email = user_dict["email"]
user.username = user_dict["email"]
user.first_name = user_dict['first_name']
user.last_name = user_dict['last_name']
user.first_name = user_dict["first_name"]
user.last_name = user_dict["last_name"]
user.save()
return user, created

24
server/vbv_lernwelt/core/migrations/0004_user_sso_id.py Normal file
View File

@ -0,0 +1,24 @@
# Generated by Django 3.2.13 on 2023-01-10 10:18
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("core", "0003_alter_user_managers"),
]
operations = [
migrations.AddField(
model_name="user",
name="sso_id",
field=models.UUIDField(
blank=True,
default=None,
null=True,
unique=True,
verbose_name="SSO subscriber ID",
),
),
]

4
server/vbv_lernwelt/core/models.py
View File

@ -15,7 +15,9 @@ class User(AbstractUser):
# objects = UserManager()
avatar_url = models.CharField(max_length=254, blank=True, default="")
email = models.EmailField("email address", unique=True)
sso_id = models.UUIDField('SSO subscriber ID', unique=True, null=True, blank=True, default=None)
sso_id = models.UUIDField(
"SSO subscriber ID", unique=True, null=True, blank=True, default=None
)
objects = UserManager()

20
server/vbv_lernwelt/course/tests/test_document_uploads.py
View File

@ -1,4 +1,4 @@
from django.conf import settings
from django.test import override_settings
from rest_framework.test import APITestCase
from vbv_lernwelt.core.create_default_users import create_default_users
@ -43,6 +43,7 @@ class DocumentUploadApiTestCase(APITestCase):
username="patrizia.huggel@eiger-versicherungen.ch", password="myvbv1234"
)
@override_settings(FILE_UPLOAD_STORAGE="s3")
def test_can_start_upload(self):
ls = LearningSequence.objects.get(
slug="test-lehrgang-lp-circle-analyse-ls-beobachten"
@ -54,16 +55,15 @@ class DocumentUploadApiTestCase(APITestCase):
self.assertEqual(response.status_code, 200)
self.assertNotEqual(response.data["url"], "")
if settings.FILE_UPLOAD_STORAGE == "s3":
self.assertTrue(response.data["url"].startswith("https://"))
self.assertEqual(
response.data["fields"]["Content-Type"], self.test_data["file_type"]
)
self.assertTrue(response.data["url"].startswith("https://"))
self.assertEqual(
response.data["fields"]["Content-Type"], self.test_data["file_type"]
)
self.assertEqual(
response.data["fields"]["Content-Disposition"],
f"attachment; filename={self.test_data['file_name']}",
)
self.assertEqual(
response.data["fields"]["Content-Disposition"],
f"attachment; filename={self.test_data['file_name']}",
)
file_id = response.data["file_id"]
file = UploadFile.objects.get(id=file_id)

3
server/vbv_lernwelt/files/integrations.py
View File

@ -119,8 +119,7 @@ def s3_delete_file(*, file_path: str):
credentials = s3_get_credentials()
s3_client = s3_get_client()
some = s3_client.delete_object(
s3_client.delete_object(
Bucket=credentials.bucket_name,
Key=file_path,
)
pass

42
server/vbv_lernwelt/files/management/commands/delete_unused_s3_files.py Normal file
View File

@ -0,0 +1,42 @@
from django.core.management.base import BaseCommand
from vbv_lernwelt.files.integrations import s3_get_client, s3_get_credentials
from vbv_lernwelt.files.models import UploadFile
class Command(BaseCommand):
help = "Delete unused files from S3"
def add_arguments(self, parser):
parser.add_argument(
"--dry-run",
action="store_true",
dest="dry_run",
default=False,
help="Dry run",
)
def handle(self, *args, **options):
dry_run = options["dry_run"]
credentials = s3_get_credentials()
s3_client = s3_get_client()
num_deleted = 0
if dry_run:
print("------ DRY RUN -------")
result = s3_client.list_objects_v2(
Bucket=credentials.bucket_name, Prefix="circledocuments/"
)
for contents in result.get("Contents"):
try:
UploadFile.objects.get(file=contents["Key"])
except UploadFile.DoesNotExist:
print(f"Deleting {contents['Key']}")
if not dry_run:
s3_client.delete_object(
Bucket=credentials.bucket_name, Key=contents["Key"]
)
num_deleted += 1
print(f"Deleted {num_deleted} files in bucket {credentials.bucket_name}")

13
server/vbv_lernwelt/sso/views.py
View File

@ -1,10 +1,7 @@
import json
import structlog as structlog
from authlib.integrations.base_client import OAuthError
from django.conf import settings
from django.contrib.auth import get_user_model, login as dj_login
from django.http import HttpResponse
from django.shortcuts import redirect
from sentry_sdk import capture_exception
@ -30,7 +27,6 @@ def authorize(request):
request
)
deocded_token = decode_jwt(token["id_token"])
return HttpResponse(json.dumps(deocded_token))
except OAuthError as e:
logger.error(f"OAuth error: {e}")
if not settings.DEBUG:
@ -38,12 +34,7 @@ def authorize(request):
return redirect(f"/{OAUTH_FAIL_REDIRECT}?state=someerror") # to be defined
user_data = _user_data_from_token_data(deocded_token)
user, created = get_user_model().objects.create_or_update_by_email(
user_data["email"],
user_data["first_name"],
user_data["last_name"],
user_data["username"],
)
user, created = get_user_model().objects.create_or_update_by_email(user_data)
dj_login(request, user)
# todo: redirect to other page if new user
@ -57,5 +48,5 @@ def _user_data_from_token_data(token: dict) -> dict:
"last_name": token.get("family_name", ""),
"username": token.get("preferred_username", first_email),
"email": first_email,
"sub": token.get("sub")
"oid": token.get("oid"),
}