Add delete unreferenced file command

client/src/components/learningPath/DocumentUploadForm.vue

@@ -12,16 +12,16 @@ interface Props {
 const { t } = useI18n();
 
 const props = withDefaults(defineProps<Props>(), {
-  learningSequences: [],
+  learningSequences: () => [],
   showUploadErrorMessage: false,
 });
 
 const emit = defineEmits<{
-  (e: "formSubmit", data: object): void;
+  (e: "formSubmit", data: DocumentUploadData): void;
 }>();
 
 const formData = reactive<DocumentUploadData>({
-  file: null | File,
+  file: null,
   name: "",
   learningSequence: {
     id: -1,
@@ -36,8 +36,12 @@ const formErrors = reactive({
 });
 
 function fileChange(e: Event) {
-  const keys = Object.keys(e.target.files);
+  const target = e.target as HTMLInputElement;
-  formData.file = keys.length > 0 ? e.target.files[keys[0]] : null;
+  if (target === null || target.files === null) {
+    return;
+  }
+
+  formData.file = target.files.length > 0 ? target.files[0] : null;
 }
 
 function submitForm() {
@@ -49,7 +53,7 @@ function submitForm() {
 }
 
 function validateForm() {
-  formErrors.file = formData.file === 0;
+  formErrors.file = formData.file === null;
   formErrors.learningSequence = formData.learningSequence.id === -1;
   formErrors.name = formData.name === "";
 
@@ -62,7 +66,7 @@ function validateForm() {
 }
 
 function resetFormErrors() {
-  for (const [_name, value] of Object.entries(formErrors)) {
+  for (let [_name, value] of Object.entries(formErrors)) {
     value = false;
   }
 }

client/src/fetchHelpers.ts

@@ -39,6 +39,10 @@ export const itPost = (url: RequestInfo, data: unknown, options: RequestInit = {
     options
   );
 
+  if (options.method === undefined) {
+    options.method = "POST";
+  }
+
   // @ts-ignore
   options.headers["X-CSRFToken"] = getCookieValue("csrftoken");
 

client/src/pages/learningPath/CirclePage.vue

@@ -110,9 +110,12 @@ onMounted(async () => {
 async function uploadDocument(data: DocumentUploadData) {
   showUploadErrorMessage.value = false;
   try {
+    if (!courseSessionsStore.courseSessionForRoute) {
+      throw new Error("No course session found");
+    }
     const newDocument = await uploadCircleDocument(
       data,
-      courseSessionsStore.courseSessionForRoute?.id
+      courseSessionsStore.courseSessionForRoute.id
     );
     const courseSessionStore = useCourseSessionsStore();
     courseSessionStore.addDocument(newDocument);
@@ -215,14 +218,23 @@ async function uploadDocument(data: DocumentUploadData) {
                     <h3 class="text-blue-dark">
                       {{ $t("circlePage.documents.title") }}
                     </h3>
-                    <ol v-if="courseSessionsStore.circleDocuments?.length > 0">
+                    <ol
+                      v-if="
+                        courseSessionsStore &&
+                        courseSessionsStore.circleDocuments &&
+                        courseSessionsStore.circleDocuments.length > 0
+                      "
+                    >
                       <li
                         v-for="learningSequence of courseSessionsStore.circleDocuments"
                         :key="learningSequence.id"
                       >
                         <h4 class="text-bold mt-4">{{ learningSequence.title }}</h4>
                         <ul>
-                          <li v-for="document of learningSequence.documents">
+                          <li
+                            v-for="document of learningSequence.documents"
+                            :key="document.url"
+                          >
                             <a :href="document.url" download>
                               <span>{{ document.name }}</span>
                             </a>

client/src/services/files.ts

@@ -2,7 +2,15 @@ import { itDelete, itFetch, itPost } from "@/fetchHelpers";
 import { getCookieValue } from "@/router/guards";
 import type { CircleDocument, DocumentUploadData } from "@/types";
 
+type FileData = {
+  fields: Record<string, string>;
+  url: string;
+};
+
 async function startFileUpload(fileData: DocumentUploadData, courseSessionId: number) {
+  if (fileData === null || fileData.file === null) {
+    return null;
+  }
   return await itPost(`/api/core/document/start/`, {
     file_type: fileData.file.type,
     file_name: fileData.file.name,
@@ -12,7 +20,7 @@ async function startFileUpload(fileData: DocumentUploadData, courseSessionId: nu
   });
 }
 
-function uploadFile(fileData, file: File) {
+function uploadFile(fileData: FileData, file: File) {
   if (fileData.fields) {
     return s3Upload(fileData, file);
   } else {
@@ -20,7 +28,7 @@ function uploadFile(fileData, file: File) {
   }
 }
 
-function directUpload(fileData, file: File) {
+function directUpload(fileData: FileData, file: File) {
   const formData = new FormData();
   formData.append("file", file);
 
@@ -39,7 +47,7 @@ function directUpload(fileData, file: File) {
   handleUpload(fileData.url, options);
 }
 
-function s3Upload(fileData, file: File) {
+function s3Upload(fileData: FileData, file: File) {
   const formData = new FormData();
   for (const [name, value] of Object.entries(fileData.fields)) {
     formData.append(name, value);
@@ -55,7 +63,7 @@ function s3Upload(fileData, file: File) {
   return handleUpload(fileData.url, options);
 }
 
-function handleUpload(url: string, options) {
+function handleUpload(url: string, options: RequestInit) {
   return itFetch(url, options).then((response) => {
     return response.json().catch(() => {
       return Promise.resolve(null);
@@ -67,6 +75,10 @@ export async function uploadCircleDocument(
   data: DocumentUploadData,
   courseSessionId: number
 ): Promise<CircleDocument> {
+  if (data.file === null) {
+    throw new Error("No file selected");
+  }
+
   const startData = await startFileUpload(data, courseSessionId);
 
   await uploadFile(startData, data.file);

client/src/stores/courseSessions.ts

@@ -1,6 +1,6 @@
 import { itGetCached, itPost } from "@/fetchHelpers";
 import { deleteCircleDocument } from "@/services/files";
-import type { CircleExpert, CourseSession, CircleDocument } from "@/types";
+import type { CircleDocument, CircleExpert, CourseSession } from "@/types";
 import _ from "lodash";
 import log from "loglevel";
 
@@ -10,6 +10,16 @@ import { useRoute } from "vue-router";
 import { useCircleStore } from "./circle";
 import { useUserStore } from "./user";
 
+export type CourseSessionsStoreState = {
+  courseSessions: CourseSession[] | undefined;
+};
+
+export type LearningSequenceCircleDocument = {
+  id: number;
+  title: string;
+  documents: CircleDocument[];
+};
+
 function loadCourseSessionsData(reload = false) {
   log.debug("loadCourseSessionsData called");
   const courseSessions = ref<CourseSession[]>([]);
@@ -17,7 +27,7 @@ function loadCourseSessionsData(reload = false) {
 
 function userExpertCircles(
   userId: number,
-  courseSessionForRoute: CourseSession
+  courseSessionForRoute: CourseSession | undefined
 ): CircleExpert[] {
   if (!courseSessionForRoute) {
     return [];
@@ -25,10 +35,6 @@ function userExpertCircles(
   return courseSessionForRoute.experts.filter((expert) => expert.user_id === userId);
 }
 
-export type CourseSessionsStoreState = {
-  courseSessions: CourseSession[] | undefined;
-};
-
   async function loadAndUpdate() {
     courseSessions.value = await itGetCached(`/api/course/sessions/`, {
       reload: reload,

client/src/types.ts

@@ -369,7 +369,7 @@ export interface ExpertSessionUser extends CourseSessionUser {
 
 // document upload
 export interface DocumentUploadData {
-  file: File;
+  file: File | null;
   name: string;
   learningSequence: {
     id: number;

server/config/settings/base.py

@@ -573,6 +573,29 @@ GRAPPLE = {
     "APPS": ["core", "course", "learnpath", "competence", "media_library"],
 }
 
+# S3 BUCKET CONFIGURATION
+FILE_UPLOAD_STORAGE = env("FILE_UPLOAD_STORAGE", default="local")  # local | s3
+
+if FILE_UPLOAD_STORAGE == "local":
+    FILE_MAX_SIZE = env.int("FILE_MAX_SIZE", default=5242880)
+
+if FILE_UPLOAD_STORAGE == "s3":
+    # Using django-storages
+    # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html
+    DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+
+    AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+    AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME")
+    AWS_S3_SIGNATURE_VERSION = env("AWS_S3_SIGNATURE_VERSION", default="s3v4")
+    FILE_MAX_SIZE = env.int("FILE_MAX_SIZE", default=5242880)  # 5MB
+
+    # https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
+    AWS_DEFAULT_ACL = env("AWS_DEFAULT_ACL", default="private")
+
+    AWS_PRESIGNED_EXPIRY = env.int("AWS_PRESIGNED_EXPIRY", default=60)  # seconds
+
 if APP_ENVIRONMENT == "development":
     # http://whitenoise.evans.io/en/latest/django.html#using-whitenoise-in-development
     INSTALLED_APPS = ["whitenoise.runserver_nostatic"] + INSTALLED_APPS  # noqa F405
@@ -607,28 +630,6 @@ if APP_ENVIRONMENT == "development":
     # https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration
     INSTALLED_APPS += ["django_extensions", "django_watchfiles"]  # noqa F405
 
-    # S3 BUCKET CONFIGURATION
-    FILE_UPLOAD_STORAGE = env("FILE_UPLOAD_STORAGE", default="local")  # local | s3
-
-    if FILE_UPLOAD_STORAGE == "local":
-        FILE_MAX_SIZE = env.int("FILE_MAX_SIZE", default=5242880)
-
-    if FILE_UPLOAD_STORAGE == "s3":
-        # Using django-storages
-        # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html
-        DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-
-        AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
-        AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
-        AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
-        AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME")
-        AWS_S3_SIGNATURE_VERSION = env("AWS_S3_SIGNATURE_VERSION", default="s3v4")
-        FILE_MAX_SIZE = env.int("FILE_MAX_SIZE", default=5242880)  # 5MB
-
-        # https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
-        AWS_DEFAULT_ACL = env("AWS_DEFAULT_ACL", default="private")
-
-        AWS_PRESIGNED_EXPIRY = env.int("AWS_PRESIGNED_EXPIRY", default=60)  # seconds
 if APP_ENVIRONMENT in ["production", "caprover"] or APP_ENVIRONMENT.startswith(
     "caprover"
 ):

server/vbv_lernwelt/course/tests/test_document_uploads.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from rest_framework.test import APITestCase
 
 from vbv_lernwelt.core.create_default_users import create_default_users
@@ -52,13 +53,17 @@ class DocumentUploadApiTestCase(APITestCase):
 
         self.assertEqual(response.status_code, 200)
         self.assertNotEqual(response.data["url"], "")
-        self.assertEqual(
+
-            response.data["fields"]["Content-Type"], self.test_data["file_type"]
+        if settings.FILE_UPLOAD_STORAGE == "s3":
-        )
+            self.assertTrue(response.data["url"].startswith("https://"))
-        self.assertEqual(
+            self.assertEqual(
-            response.data["fields"]["Content-Disposition"],
+                response.data["fields"]["Content-Type"], self.test_data["file_type"]
-            f"attachment; filename={self.test_data['file_name']}",
+            )
-        )
+
+            self.assertEqual(
+                response.data["fields"]["Content-Disposition"],
+                f"attachment; filename={self.test_data['file_name']}",
+            )
 
         file_id = response.data["file_id"]
         file = File.objects.get(id=file_id)

server/vbv_lernwelt/files/integrations.py

@@ -113,3 +113,14 @@ def s3_generate_presigned_url(*, file_path: str) -> str:
         Params={"Bucket": credentials.bucket_name, "Key": file_path},
         ExpiresIn=credentials.presigned_expiry,
     )
+
+
+def s3_delete_file(*, file_path: str):
+    credentials = s3_get_credentials()
+    s3_client = s3_get_client()
+
+    some = s3_client.delete_object(
+        Bucket=credentials.bucket_name,
+        Key=file_path,
+    )
+    pass

server/vbv_lernwelt/files/management/commands/delete_unreferenced_files.py

@@ -0,0 +1,47 @@
+from django.core.management.base import BaseCommand
+
+from vbv_lernwelt.files.models import File
+
+
+class Command(BaseCommand):
+    help = "Delete unreferenced uploads and delete their files"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            dest="dry_run",
+            default=False,
+            help="Dry run",
+        )
+
+    def handle(self, *args, **options):
+        dry_run = options["dry_run"]
+
+        num_deleted = 0
+
+        unreferenced_uploads = File.objects.filter(upload_finished_at__isnull=True)
+
+        if dry_run:
+            print("------ DRY RUN -------")
+
+        print(
+            "Going to delete {} unreferenced uploads".format(
+                unreferenced_uploads.count()
+            )
+        )
+        for upload in unreferenced_uploads:
+            try:
+                if not dry_run:
+                    upload.delete_file()
+                    file_id = upload.id
+                    upload.delete()
+                    print("Deleted file with id {}".format(file_id))
+                else:
+                    print("Would delete file with id {}".format(upload.id))
+                num_deleted += 1
+            except Exception as e:
+                print(e)
+                pass
+
+        print("Deleted {:d} uploads".format(num_deleted))

server/vbv_lernwelt/files/models.py

@@ -3,7 +3,7 @@ from django.db import models
 
 from vbv_lernwelt.core.models import User
 from vbv_lernwelt.files.enums import FileUploadStorage
-from vbv_lernwelt.files.integrations import s3_generate_presigned_url
+from vbv_lernwelt.files.integrations import s3_delete_file, s3_generate_presigned_url
 from vbv_lernwelt.files.utils import file_generate_upload_path
 
 
@@ -39,3 +39,9 @@ class File(models.Model):
             return s3_generate_presigned_url(file_path=str(self.file))
 
         return f"{self.file.url}"
+
+    def delete_file(self):
+        if settings.FILE_UPLOAD_STORAGE == FileUploadStorage.S3.value:
+            return s3_delete_file(file_path=str(self.file))
+        else:
+            return self.file.delete()