feat: chunked uploads

Dockerfile

@@ -4,7 +4,7 @@ MAINTAINER Computer Science House <rtp@csh.rit.edu>
 ENV IMAGEIO_USERDIR /var/lib/gallery
 
 RUN apt-get update && \
-    apt-get install -y libldap-dev libsasl2-dev libmagic-dev ghostscript libldap-common && \
+    apt-get install -y libldap-dev libsasl2-dev libmagic-dev ghostscript libldap-common imagemagick libheif1 libheif-dev libraw-dev libraw20 dcraw && \ 
     apt-get autoremove --yes && \
     apt-get clean autoclean && \
     sed -i \

gallery/__init__.py

@@ -40,6 +40,7 @@
 import flask_migrate
 import requests
 from werkzeug.utils import secure_filename
+import shutil
 
 app = Flask(__name__)
 app.config.update({
@@ -866,7 +867,11 @@ def display_thumbnail(file_id: int, auth_dict: Optional[Dict[str, Any]] = None):
 
     file_model = File.query.filter(File.id == file_id).first()
 
-    link = storage_interface.get_link("thumbnails/{}".format(file_model.s3_id))
+    thumbnail_uuid = file_model.thumbnail_uuid
+    if len(thumbnail_uuid.split('.')) > 1:
+        thumbnail_uuid = thumbnail_uuid.split('.')[0]
+
+    link = storage_interface.get_link("thumbnails/{}".format(thumbnail_uuid))
     if "LOCAL_STORAGE_PATH" in app.config:
         link = "http://" + app.config["SERVER_NAME"] + link
     req = requests.get(link)
@@ -1052,7 +1057,7 @@ def render_dir(dir_id: int, auth_dict: Optional[Dict[str, Any]] = None):
     dir_model = Directory.query.filter(Directory.id == dir_id).first()
     if dir_model is None:
         abort(404)
-    description = dir_model.description
+    description = dir_model.description or ""
     display_description = len(description) > 0
 
     display_parent = True
@@ -1100,7 +1105,7 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None):
     if gallery_lockdown and (not auth_dict['is_eboard'] and not auth_dict['is_rtp']):
         abort(405)
 
-    description = file_model.caption
+    description = file_model.caption or ""
     display_description = len(description) > 0
     display_parent = True
     if file_model is None or file_model.parent is None:
@@ -1135,6 +1140,106 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None):
                            lockdown=gallery_lockdown)
 
 
+@app.route('/upload/chunk', methods=['POST'])
+@auth.oidc_auth('default')
+@gallery_auth
+def upload_chunk(auth_dict: Optional[Dict[str, Any]] = None):
+    chunk = request.files.get('gallery-upload')
+    if chunk is None:
+        return jsonify({'error': 'no chunk'}), 400
+
+    dz_uuid      = request.form.get('dzuuid')
+    chunk_index  = int(request.form.get('dzchunkindex', 0))
+    chunk_size   = int(request.form.get('dzchunksize', 0))
+    filename     = secure_filename(request.form.get('dzfilename', ''))
+
+    if not dz_uuid or not filename:
+        return jsonify({'error': 'missing chunk metadata'}), 400
+
+    chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid)
+    os.makedirs(chunk_dir, exist_ok=True)
+
+    out_path = os.path.join(chunk_dir, 'assembled')
+    with open(out_path, 'ab') as out:
+        out.write(chunk.read())
+
+    return jsonify({'status': 'ok', 'chunk': chunk_index}), 200
+
+
+@app.route('/upload/chunk/finalize', methods=['POST'])
+@auth.oidc_auth('default')
+@gallery_auth
+def finalize_upload(auth_dict: Optional[Dict[str, Any]] = None):
+    assert auth_dict is not None
+    owner     = auth_dict['uuid']
+    parent    = request.form.get('parent_id')
+    dz_uuid   = request.form.get('dzuuid')
+    filename  = secure_filename(request.form.get('filename', ''))
+    total_chunks = int(request.form.get('dztotalchunkcount', 0))
+
+    if not all([parent, dz_uuid, filename, total_chunks]):
+        return jsonify({'error': 'missing parameters'}), 400
+
+    chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid)
+
+    upload_status: Dict[str, Any] = {}
+    upload_status['redirect'] = '/view/dir/' + str(parent)
+    errors: List[str] = []
+    success: List[Dict[str, Any]] = []
+
+    file_model = File.query.filter(File.parent == parent) \
+                           .filter(File.name == filename).first()
+    if file_model is not None:
+        errors.append(filename)
+        upload_status['error'] = errors
+        upload_status['success'] = success
+        return jsonify(upload_status), 200
+
+    dir_path = tempfile.mkdtemp()
+    filepath = os.path.join(dir_path, filename)
+    try:
+        assembled_path = os.path.join(chunk_dir, 'assembled')
+        if not os.path.exists(assembled_path):
+            return jsonify({'error': 'assembled file missing'}), 400
+        shutil.move(assembled_path, filepath)
+        try:
+            mime, file_model = add_file(filename, dir_path, parent, '', owner)
+        except OSError as e:
+            if e.errno == 28:
+                return jsonify({'error': 'storage full'}), 507
+            raise
+        if file_model is None:
+            errors.append(filename)
+        else:
+            with open(filepath, 'rb') as f_hnd:
+                storage_interface.put(
+                    'files/{}'.format(file_model.s3_id),
+                    f_hnd,
+                    filename,
+                    mime
+                )
+            os.remove(filepath)
+
+            thumb_path = os.path.join(dir_path, file_model.thumbnail_uuid)
+            with open(thumb_path, 'rb') as f_hnd:
+                storage_interface.put(
+                    'thumbnails/' + file_model.s3_id,
+                    f_hnd,
+                    'thumb_' + filename + '.' + thumb_path.split('.')[-1],
+                    'image/gif' if thumb_path.endswith('.gif') else 'image/jpeg'
+                )
+            os.remove(thumb_path)
+            success.append({'name': file_model.name, 'id': file_model.id})
+    finally:
+        shutil.rmtree(chunk_dir, ignore_errors=True)
+        shutil.rmtree(dir_path, ignore_errors=True)
+
+    upload_status['error'] = errors
+    upload_status['success'] = success
+    refresh_default_thumbnails()
+    return jsonify(upload_status), 200
+
+
 @app.route("/view/random_file")
 @auth.oidc_auth('default')
 def get_random_file():

gallery/file_modules/__init__.py

@@ -3,6 +3,7 @@
 from typing import Any, Dict, List, Optional, Tuple
 from wand.image import Image
 from wand.color import Color
+import rawpy
 
 from gallery.util import DEFAULT_THUMBNAIL_NAME
 from gallery.util import hash_file
@@ -65,6 +66,10 @@ def generate_thumbnail(self):
 from gallery.file_modules.txt import TXTFile
 from gallery.file_modules.mp3 import MP3File
 from gallery.file_modules.wav import WAVFile
+from gallery.file_modules.heic import HEICFile
+from gallery.file_modules.nef import NEFFile
+from gallery.file_modules.mov import MOVFile
+from gallery.file_modules.m4a import M4AFile
 
 file_mimetype_relation = {
     "image/jpeg": JPEGFile,
@@ -76,20 +81,36 @@ def generate_thumbnail(self):
     "image/x-windows-bmp": BMPFile,
     "image/tiff": TIFFFile,
     "image/x-tiff": TIFFFile,
+    "image/heic": HEICFile,
+    "image/x-nikon-nef": NEFFile,
     "video/mp4": MP4File,
     "video/webm": WebMFile,
     "video/ogg": OggFile,
+    "video/quicktime": MOVFile,
     "application/pdf": PDFFile,
     "text/plain": TXTFile,
     "audio/mpeg": MP3File,
-    "audio/x-wav": WAVFile
+    "audio/x-wav": WAVFile,
+    "audio/mp4": M4AFile,
+    "audio/x-m4a": M4AFile,
 }
 
 
 # classism
 def parse_file_info(file_path: str, dir_path: str) -> Tuple[str, Optional[FileModule]]:
     print("entering parse_file_info")
+
     mime_type = magic.from_file(file_path, mime=True)
+
+    if "tif" in mime_type: # .nef is a special case, magic reads it as .tiff, but it is not processed correctly by the tiff module
+        # check if it is a raw file
+        try:
+            with rawpy.imread(file_path): # this may cause issues for other raw files
+                mime_type = "image/x-nikon-nef"
+        except rawpy.LibRawFileUnsupportedError:
+            pass
+        except rawpy.LibRawIOError:
+            pass
     print(mime_type)
     print(file_path)
 

gallery/file_modules/heic.py

@@ -0,0 +1,30 @@
+import os
+from PIL import Image as PILImage
+import pillow_heif
+
+from gallery.file_modules import FileModule
+from gallery.util import hash_file
+
+pillow_heif.register_heif_opener()
+
+class HEICFile(FileModule):
+    def __init__(self, file_path, dir_path):
+        FileModule.__init__(self, file_path, dir_path)
+        self.mime_type = "image/heic"
+
+        self.generate_thumbnail()
+
+    def generate_thumbnail(self):
+        self.thumbnail_uuid = hash_file(self.file_path) + ".jpg"
+
+        thumb_path = os.path.join(self.dir_path, self.thumbnail_uuid)
+
+        img = PILImage.open(self.file_path).convert("RGB")
+
+        size = min(img.width, img.height)
+        left = (img.width - size) // 2
+        top = (img.height - size) // 2
+        img = img.crop((left, top, left + size, top + size))
+
+        img = img.resize((256, 256))
+        img.save(thumb_path, "JPEG")

gallery/file_modules/m4a.py

@@ -0,0 +1,18 @@
+import os
+from wand.image import Image
+
+from gallery.file_modules import FileModule
+from gallery.util import hash_file
+
+class M4AFile(FileModule):
+    def __init__(self, file_path, dir_path):
+        FileModule.__init__(self, file_path, dir_path)
+        self.mime_type = "audio/mp4"
+
+        self.generate_thumbnail()
+
+    def generate_thumbnail(self):
+        self.thumbnail_uuid = hash_file(self.file_path) + ".jpg"
+
+        with Image(filename="thumbnails/reedphoto.jpg") as bg:
+                bg.save(filename=os.path.join(self.dir_path, self.thumbnail_uuid))

gallery/file_modules/mov.py

@@ -0,0 +1,33 @@
+from moviepy.editor import VideoFileClip
+import os
+from wand.image import Image
+from wand.color import Color
+
+from gallery.file_modules import FileModule
+from gallery.util import hash_file
+
+
+class MOVFile(FileModule):
+
+    def __init__(self, file_path, dir_path):
+        FileModule.__init__(self, file_path, dir_path)
+        self.mime_type = "video/quicktime"
+
+        self.generate_thumbnail()
+
+    def generate_thumbnail(self):
+        self.thumbnail_uuid = hash_file(self.file_path) + ".jpg"
+        thumbnail_loc = os.path.join(self.dir_path, self.thumbnail_uuid)
+
+        clip = VideoFileClip(self.file_path)
+        time_mark = clip.duration * 0.05
+        clip.save_frame(thumbnail_loc, t=time_mark)
+
+        with Image(filename=thumbnail_loc) as img:
+            with img.clone() as image:
+                size = image.width if image.width < image.height else image.height
+                image.crop(width=size, height=size, gravity='center')
+                image.resize(256, 256)
+                image.background_color = Color("#EEEEEE")
+                image.format = 'jpeg'
+                image.save(filename=thumbnail_loc)

gallery/file_modules/nef.py

@@ -0,0 +1,29 @@
+import os
+import rawpy
+import imageio
+
+from gallery.file_modules import FileModule
+from gallery.util import hash_file
+
+
+class NEFFile(FileModule):
+    def __init__(self, file_path, dir_path):
+        FileModule.__init__(self, file_path, dir_path)
+        self.mime_type = "image/x-nikon-nef"
+
+        self.generate_thumbnail()
+
+    def generate_thumbnail(self):
+        self.thumbnail_uuid = hash_file(self.file_path) + ".jpg"
+        thumb_path = os.path.join(self.dir_path, self.thumbnail_uuid)
+
+        with rawpy.imread(self.file_path) as raw:
+            rgb = raw.postprocess(output_bps=8)
+
+            h, w, _ = rgb.shape
+            size = min(h, w)
+            y = (h - size) // 2
+            x = (w - size) // 2
+            rgb = rgb[y:y+size, x:x+size]
+
+            imageio.imwrite(thumb_path, rgb)

gallery/templates/view_dir.html

@@ -245,34 +245,60 @@ <h4 class="modal-title">Upload</h4>
 Dropzone.autoDiscover = false;
 
     $(function() {
-
+        
         var myDropzone = new Dropzone('div#galleryUpload', {
-            url: '/upload',
+            url: '/upload/chunk',
             paramName: "gallery-upload",
             uploadMultiple: false,
             autoProcessQueue: false,
             parallelUploads: 3,
-            success: afterUpload,
-            maxFilesize: 1024,
+            maxFilesize: 5 * 1024,
+            chunking: true,
+            forceChunking: true,
+            chunkSize: 10 * 1024 * 1024,
+            retryChunks: true,
+            retryChunksLimit: 3,
+            timeout: 0,
             init: function() {
                 this.on('addedfile', function(file) {
                     console.log(file.name + ": " + file.size);
-                    if (file.size > (1024 * 1024 * 1024)) {
-                        var warning = "<div class='alert alert-dismissible alert-danger' id='upload-alert'><button type='button' class='close' data-dismiss='alert'>&times;</button><span class='glyphicon glyphicon-exclamation-sign'></span> File \"<strong>" + file.name + "</strong>\" is too large, max filesize is 1GB.</div>";
-                        $('#upload').after(warning);
-                        this.removeFile(file);
-                    }
                 });
 
                 this.on('accept', function(file, done) {
                     done();
                 });
 
                 this.on('sending', function(file, xhr, formData) {
-                    formData.append('parent_id', "{{directory.id}}");
+                    formData.append('parent_id', '{{ directory.id }}');
+                    formData.append('dzfilename', file.name);
+                });
+
+                this.on('success', function(file, response) {
+                    if (file.upload.chunked && file.upload.chunkIndex < file.upload.totalChunkCount - 1) {
+                        return;
+                    }
+                    var formData = new FormData();
+                    formData.append('parent_id', '{{ directory.id }}');
+                    formData.append('dzuuid', file.upload.uuid);
+                    formData.append('filename', file.name);
+                    formData.append('dztotalchunkcount', file.upload.totalChunkCount);
+                    fetch('/upload/chunk/finalize', {
+                        method: 'POST',
+                        body: formData,
+                    })
+                    .then(function(r) { return r.json(); })
+                    .then(function(data) {
+                        console.log('finalize response:', JSON.stringify(data));
+                        afterUpload(file, data);
+                    })
+                    .catch(function(err) {
+                        console.error('Finalize error:', err);
+                    });
                 });
 
-                this.on('complete', function(file) {
+                this.on('error', function(file, message) {
+                    var warning = "<div class='alert alert-dismissible alert-danger' id='upload-alert'><button type='button' class='close' data-dismiss='alert'>&times;</button><span class='glyphicon glyphicon-exclamation-sign'></span> Upload failed for <strong>" + file.name + "</strong>: " + (typeof message === 'object' ? (message.error || JSON.stringify(message)) : message) + "</div>";
+                    $('#upload').after(warning);
                     this.removeFile(file);
                 });
             }