diff --git a/Dockerfile b/Dockerfile index 219a7be..adbebee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Computer Science House ENV IMAGEIO_USERDIR /var/lib/gallery RUN apt-get update && \ - apt-get install -y libldap-dev libsasl2-dev libmagic-dev ghostscript libldap-common && \ + apt-get install -y libldap-dev libsasl2-dev libmagic-dev ghostscript libldap-common imagemagick libheif1 libheif-dev libraw-dev libraw20 dcraw && \ apt-get autoremove --yes && \ apt-get clean autoclean && \ sed -i \ diff --git a/gallery/__init__.py b/gallery/__init__.py index 36b55e4..ab916d0 100644 --- a/gallery/__init__.py +++ b/gallery/__init__.py @@ -40,6 +40,7 @@ import flask_migrate import requests from werkzeug.utils import secure_filename +import shutil app = Flask(__name__) app.config.update({ @@ -866,7 +867,11 @@ def display_thumbnail(file_id: int, auth_dict: Optional[Dict[str, Any]] = None): file_model = File.query.filter(File.id == file_id).first() - link = storage_interface.get_link("thumbnails/{}".format(file_model.s3_id)) + thumbnail_uuid = file_model.thumbnail_uuid + if len(thumbnail_uuid.split('.')) > 1: + thumbnail_uuid = thumbnail_uuid.split('.')[0] + + link = storage_interface.get_link("thumbnails/{}".format(thumbnail_uuid)) if "LOCAL_STORAGE_PATH" in app.config: link = "http://" + app.config["SERVER_NAME"] + link req = requests.get(link) @@ -1052,7 +1057,7 @@ def render_dir(dir_id: int, auth_dict: Optional[Dict[str, Any]] = None): dir_model = Directory.query.filter(Directory.id == dir_id).first() if dir_model is None: abort(404) - description = dir_model.description + description = dir_model.description or "" display_description = len(description) > 0 display_parent = True @@ -1100,7 +1105,7 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None): if gallery_lockdown and (not auth_dict['is_eboard'] and not auth_dict['is_rtp']): abort(405) - description = file_model.caption + description = file_model.caption or "" display_description = len(description) > 0 display_parent = True if file_model is None or file_model.parent is None: @@ -1135,6 +1140,106 @@ def render_file(file_id: int, auth_dict: Optional[Dict[str, Any]] = None): lockdown=gallery_lockdown) +@app.route('/upload/chunk', methods=['POST']) +@auth.oidc_auth('default') +@gallery_auth +def upload_chunk(auth_dict: Optional[Dict[str, Any]] = None): + chunk = request.files.get('gallery-upload') + if chunk is None: + return jsonify({'error': 'no chunk'}), 400 + + dz_uuid = request.form.get('dzuuid') + chunk_index = int(request.form.get('dzchunkindex', 0)) + chunk_size = int(request.form.get('dzchunksize', 0)) + filename = secure_filename(request.form.get('dzfilename', '')) + + if not dz_uuid or not filename: + return jsonify({'error': 'missing chunk metadata'}), 400 + + chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid) + os.makedirs(chunk_dir, exist_ok=True) + + out_path = os.path.join(chunk_dir, 'assembled') + with open(out_path, 'ab') as out: + out.write(chunk.read()) + + return jsonify({'status': 'ok', 'chunk': chunk_index}), 200 + + +@app.route('/upload/chunk/finalize', methods=['POST']) +@auth.oidc_auth('default') +@gallery_auth +def finalize_upload(auth_dict: Optional[Dict[str, Any]] = None): + assert auth_dict is not None + owner = auth_dict['uuid'] + parent = request.form.get('parent_id') + dz_uuid = request.form.get('dzuuid') + filename = secure_filename(request.form.get('filename', '')) + total_chunks = int(request.form.get('dztotalchunkcount', 0)) + + if not all([parent, dz_uuid, filename, total_chunks]): + return jsonify({'error': 'missing parameters'}), 400 + + chunk_dir = os.path.join(tempfile.gettempdir(), 'chonks', dz_uuid) + + upload_status: Dict[str, Any] = {} + upload_status['redirect'] = '/view/dir/' + str(parent) + errors: List[str] = [] + success: List[Dict[str, Any]] = [] + + file_model = File.query.filter(File.parent == parent) \ + .filter(File.name == filename).first() + if file_model is not None: + errors.append(filename) + upload_status['error'] = errors + upload_status['success'] = success + return jsonify(upload_status), 200 + + dir_path = tempfile.mkdtemp() + filepath = os.path.join(dir_path, filename) + try: + assembled_path = os.path.join(chunk_dir, 'assembled') + if not os.path.exists(assembled_path): + return jsonify({'error': 'assembled file missing'}), 400 + shutil.move(assembled_path, filepath) + try: + mime, file_model = add_file(filename, dir_path, parent, '', owner) + except OSError as e: + if e.errno == 28: + return jsonify({'error': 'storage full'}), 507 + raise + if file_model is None: + errors.append(filename) + else: + with open(filepath, 'rb') as f_hnd: + storage_interface.put( + 'files/{}'.format(file_model.s3_id), + f_hnd, + filename, + mime + ) + os.remove(filepath) + + thumb_path = os.path.join(dir_path, file_model.thumbnail_uuid) + with open(thumb_path, 'rb') as f_hnd: + storage_interface.put( + 'thumbnails/' + file_model.s3_id, + f_hnd, + 'thumb_' + filename + '.' + thumb_path.split('.')[-1], + 'image/gif' if thumb_path.endswith('.gif') else 'image/jpeg' + ) + os.remove(thumb_path) + success.append({'name': file_model.name, 'id': file_model.id}) + finally: + shutil.rmtree(chunk_dir, ignore_errors=True) + shutil.rmtree(dir_path, ignore_errors=True) + + upload_status['error'] = errors + upload_status['success'] = success + refresh_default_thumbnails() + return jsonify(upload_status), 200 + + @app.route("/view/random_file") @auth.oidc_auth('default') def get_random_file(): diff --git a/gallery/file_modules/__init__.py b/gallery/file_modules/__init__.py index b2a8030..a54a6e9 100644 --- a/gallery/file_modules/__init__.py +++ b/gallery/file_modules/__init__.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple from wand.image import Image from wand.color import Color +import rawpy from gallery.util import DEFAULT_THUMBNAIL_NAME from gallery.util import hash_file @@ -65,6 +66,10 @@ def generate_thumbnail(self): from gallery.file_modules.txt import TXTFile from gallery.file_modules.mp3 import MP3File from gallery.file_modules.wav import WAVFile +from gallery.file_modules.heic import HEICFile +from gallery.file_modules.nef import NEFFile +from gallery.file_modules.mov import MOVFile +from gallery.file_modules.m4a import M4AFile file_mimetype_relation = { "image/jpeg": JPEGFile, @@ -76,20 +81,36 @@ def generate_thumbnail(self): "image/x-windows-bmp": BMPFile, "image/tiff": TIFFFile, "image/x-tiff": TIFFFile, + "image/heic": HEICFile, + "image/x-nikon-nef": NEFFile, "video/mp4": MP4File, "video/webm": WebMFile, "video/ogg": OggFile, + "video/quicktime": MOVFile, "application/pdf": PDFFile, "text/plain": TXTFile, "audio/mpeg": MP3File, - "audio/x-wav": WAVFile + "audio/x-wav": WAVFile, + "audio/mp4": M4AFile, + "audio/x-m4a": M4AFile, } # classism def parse_file_info(file_path: str, dir_path: str) -> Tuple[str, Optional[FileModule]]: print("entering parse_file_info") + mime_type = magic.from_file(file_path, mime=True) + + if "tif" in mime_type: # .nef is a special case, magic reads it as .tiff, but it is not processed correctly by the tiff module + # check if it is a raw file + try: + with rawpy.imread(file_path): # this may cause issues for other raw files + mime_type = "image/x-nikon-nef" + except rawpy.LibRawFileUnsupportedError: + pass + except rawpy.LibRawIOError: + pass print(mime_type) print(file_path) diff --git a/gallery/file_modules/heic.py b/gallery/file_modules/heic.py new file mode 100644 index 0000000..93c2034 --- /dev/null +++ b/gallery/file_modules/heic.py @@ -0,0 +1,30 @@ +import os +from PIL import Image as PILImage +import pillow_heif + +from gallery.file_modules import FileModule +from gallery.util import hash_file + +pillow_heif.register_heif_opener() + +class HEICFile(FileModule): + def __init__(self, file_path, dir_path): + FileModule.__init__(self, file_path, dir_path) + self.mime_type = "image/heic" + + self.generate_thumbnail() + + def generate_thumbnail(self): + self.thumbnail_uuid = hash_file(self.file_path) + ".jpg" + + thumb_path = os.path.join(self.dir_path, self.thumbnail_uuid) + + img = PILImage.open(self.file_path).convert("RGB") + + size = min(img.width, img.height) + left = (img.width - size) // 2 + top = (img.height - size) // 2 + img = img.crop((left, top, left + size, top + size)) + + img = img.resize((256, 256)) + img.save(thumb_path, "JPEG") \ No newline at end of file diff --git a/gallery/file_modules/m4a.py b/gallery/file_modules/m4a.py new file mode 100644 index 0000000..155f3ff --- /dev/null +++ b/gallery/file_modules/m4a.py @@ -0,0 +1,18 @@ +import os +from wand.image import Image + +from gallery.file_modules import FileModule +from gallery.util import hash_file + +class M4AFile(FileModule): + def __init__(self, file_path, dir_path): + FileModule.__init__(self, file_path, dir_path) + self.mime_type = "audio/mp4" + + self.generate_thumbnail() + + def generate_thumbnail(self): + self.thumbnail_uuid = hash_file(self.file_path) + ".jpg" + + with Image(filename="thumbnails/reedphoto.jpg") as bg: + bg.save(filename=os.path.join(self.dir_path, self.thumbnail_uuid)) diff --git a/gallery/file_modules/mov.py b/gallery/file_modules/mov.py new file mode 100644 index 0000000..a99be5d --- /dev/null +++ b/gallery/file_modules/mov.py @@ -0,0 +1,33 @@ +from moviepy.editor import VideoFileClip +import os +from wand.image import Image +from wand.color import Color + +from gallery.file_modules import FileModule +from gallery.util import hash_file + + +class MOVFile(FileModule): + + def __init__(self, file_path, dir_path): + FileModule.__init__(self, file_path, dir_path) + self.mime_type = "video/quicktime" + + self.generate_thumbnail() + + def generate_thumbnail(self): + self.thumbnail_uuid = hash_file(self.file_path) + ".jpg" + thumbnail_loc = os.path.join(self.dir_path, self.thumbnail_uuid) + + clip = VideoFileClip(self.file_path) + time_mark = clip.duration * 0.05 + clip.save_frame(thumbnail_loc, t=time_mark) + + with Image(filename=thumbnail_loc) as img: + with img.clone() as image: + size = image.width if image.width < image.height else image.height + image.crop(width=size, height=size, gravity='center') + image.resize(256, 256) + image.background_color = Color("#EEEEEE") + image.format = 'jpeg' + image.save(filename=thumbnail_loc) diff --git a/gallery/file_modules/nef.py b/gallery/file_modules/nef.py new file mode 100644 index 0000000..b880636 --- /dev/null +++ b/gallery/file_modules/nef.py @@ -0,0 +1,29 @@ +import os +import rawpy +import imageio + +from gallery.file_modules import FileModule +from gallery.util import hash_file + + +class NEFFile(FileModule): + def __init__(self, file_path, dir_path): + FileModule.__init__(self, file_path, dir_path) + self.mime_type = "image/x-nikon-nef" + + self.generate_thumbnail() + + def generate_thumbnail(self): + self.thumbnail_uuid = hash_file(self.file_path) + ".jpg" + thumb_path = os.path.join(self.dir_path, self.thumbnail_uuid) + + with rawpy.imread(self.file_path) as raw: + rgb = raw.postprocess(output_bps=8) + + h, w, _ = rgb.shape + size = min(h, w) + y = (h - size) // 2 + x = (w - size) // 2 + rgb = rgb[y:y+size, x:x+size] + + imageio.imwrite(thumb_path, rgb) \ No newline at end of file diff --git a/gallery/templates/view_dir.html b/gallery/templates/view_dir.html index 3bc1c66..331702c 100644 --- a/gallery/templates/view_dir.html +++ b/gallery/templates/view_dir.html @@ -245,23 +245,23 @@ Dropzone.autoDiscover = false; $(function() { - + var myDropzone = new Dropzone('div#galleryUpload', { - url: '/upload', + url: '/upload/chunk', paramName: "gallery-upload", uploadMultiple: false, autoProcessQueue: false, parallelUploads: 3, - success: afterUpload, - maxFilesize: 1024, + maxFilesize: 5 * 1024, + chunking: true, + forceChunking: true, + chunkSize: 10 * 1024 * 1024, + retryChunks: true, + retryChunksLimit: 3, + timeout: 0, init: function() { this.on('addedfile', function(file) { console.log(file.name + ": " + file.size); - if (file.size > (1024 * 1024 * 1024)) { - var warning = "
File \"" + file.name + "\" is too large, max filesize is 1GB.
"; - $('#upload').after(warning); - this.removeFile(file); - } }); this.on('accept', function(file, done) { @@ -269,10 +269,36 @@ }); this.on('sending', function(file, xhr, formData) { - formData.append('parent_id', "{{directory.id}}"); + formData.append('parent_id', '{{ directory.id }}'); + formData.append('dzfilename', file.name); + }); + + this.on('success', function(file, response) { + if (file.upload.chunked && file.upload.chunkIndex < file.upload.totalChunkCount - 1) { + return; + } + var formData = new FormData(); + formData.append('parent_id', '{{ directory.id }}'); + formData.append('dzuuid', file.upload.uuid); + formData.append('filename', file.name); + formData.append('dztotalchunkcount', file.upload.totalChunkCount); + fetch('/upload/chunk/finalize', { + method: 'POST', + body: formData, + }) + .then(function(r) { return r.json(); }) + .then(function(data) { + console.log('finalize response:', JSON.stringify(data)); + afterUpload(file, data); + }) + .catch(function(err) { + console.error('Finalize error:', err); + }); }); - this.on('complete', function(file) { + this.on('error', function(file, message) { + var warning = "
Upload failed for " + file.name + ": " + (typeof message === 'object' ? (message.error || JSON.stringify(message)) : message) + "
"; + $('#upload').after(warning); this.removeFile(file); }); } diff --git a/gallery/templates/view_file.html b/gallery/templates/view_file.html index 1dc49b7..8edccf0 100644 --- a/gallery/templates/view_file.html +++ b/gallery/templates/view_file.html @@ -59,7 +59,7 @@ {% elif file.mimetype.split('/')[0] == "video" %} {% elif file.mimetype == "application/pdf" or file.mimetype == "text/plain" %} diff --git a/requirements.txt b/requirements.txt index a5b9de1..ed75835 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,3 +70,8 @@ Werkzeug==3.1.3 wrapt==1.17.2 xmltodict==0.14.2 zipp==3.19.1 +# NEF and HEIC support +pillow==11.1.0 +pillow_heif==1.2.0 +rawpy==0.26.1 +imageio==2.4.0 \ No newline at end of file