chore: tweaks to bulk uploads

Author: MrYuion

src/app/overlays/bulk-item-modal/bulk-item-modal.component.ts

@@ -16,6 +16,7 @@ import {
 } from '@placeos/ts-client';
 import { Observable } from 'rxjs';
 import { unique } from '../../common/general';
+import { notifyError, notifyWarn } from '../../common/notifications';
 import { HashMap, Identity } from '../../common/types';
 import { IconComponent } from '../../ui/icon.component';
 import { TranslatePipe } from '../../ui/translate.pipe';
@@ -150,13 +151,16 @@ export class BulkItemModalComponent<
     /** List of raw data to use for bulk add */
     public data_list: HashMap<unknown>[] = [];
     /** Whether requests are being processed */
-    public loading: boolean;
+    public loading = false;
     /** Template data for use */
     public template: HashMap<unknown>[] = [];
     public mappings: Record<string, string> = {};
 
     public available_fields: Identity[] = [];
 
+    /** Required fields for validation depending on resource type */
+    private _required_fields: string[] = [];
+
     public get type(): string {
         return this._data.name;
     }
@@ -168,6 +172,7 @@ export class BulkItemModalComponent<
     constructor() {
         this.available_fields = this.getAvailableFields();
         this.template = this.generateTemplate();
+        this._required_fields = this.getRequiredFields();
     }
 
     /**
@@ -179,15 +184,9 @@ export class BulkItemModalComponent<
             if (is_mapped) {
                 const Resource = this._data.constr;
                 this.item_list = data.map((item) => {
-                    const new_item: HashMap<unknown> = {};
-                    Object.keys(item).forEach((key) => {
-                        try {
-                            new_item[key] = JSON.parse(item[key] as string);
-                        } catch {
-                            new_item[key] = item[key];
-                        }
-                    });
-                    return new Resource(new_item);
+                    // Values are already parsed by csvToJson/parseCSV,
+                    // so no need for additional JSON.parse here
+                    return new Resource(item);
                 });
                 this.flow_step = 'list';
             } else {
@@ -198,11 +197,35 @@ export class BulkItemModalComponent<
     }
 
     public showStatus() {
+        const validation_errors = this.validateItems();
+        if (validation_errors.length > 0) {
+            for (const error of validation_errors.slice(0, 5)) {
+                notifyError(error);
+            }
+            if (validation_errors.length > 5) {
+                notifyWarn(
+                    `...and ${validation_errors.length - 5} more validation errors.`,
+                );
+            }
+            return;
+        }
+        const duplicates = this.findDuplicates();
+        if (duplicates.length > 0) {
+            for (const warning of duplicates.slice(0, 5)) {
+                notifyWarn(warning);
+            }
+            if (duplicates.length > 5) {
+                notifyWarn(
+                    `...and ${duplicates.length - 5} more duplicate warnings.`,
+                );
+            }
+        }
         this.flow_step = 'status';
     }
 
     public done() {
-        setTimeout(() => this._dialog_ref.close(), 3000);
+        this.loading = false;
+        setTimeout(() => this._dialog_ref.close(), 5000);
     }
 
     private getAvailableFields(): Identity[] {
@@ -233,4 +256,57 @@ export class BulkItemModalComponent<
                 return [new PlaceZone(ZONE_TEMPLATE).toJSON()];
         }
     }
+
+    private getRequiredFields(): string[] {
+        switch (this._data.constr as unknown) {
+            case PlaceUser:
+                return ['email', 'authority_id'];
+            case PlaceSystem:
+                return ['name'];
+            case PlaceModule:
+                return ['driver_id'];
+            case PlaceDriver:
+                return ['name', 'module_name', 'role'];
+            case PlaceZone:
+                return ['name'];
+            default:
+                return [];
+        }
+    }
+
+    private validateItems(): string[] {
+        const errors: string[] = [];
+        this.item_list.forEach((item, index) => {
+            for (const field of this._required_fields) {
+                const value = item[field];
+                if (value === undefined || value === null || value === '') {
+                    errors.push(
+                        `Row ${index + 1}: Missing required field "${field}"`,
+                    );
+                }
+            }
+        });
+        return errors;
+    }
+
+    private findDuplicates(): string[] {
+        const warnings: string[] = [];
+        const seen_keys = new Map<string, number>();
+        const dedup_field =
+            this._data.constr === (PlaceUser as unknown) ? 'email' : 'name';
+
+        this.item_list.forEach((item, index) => {
+            const key = String(item[dedup_field] || '')
+                .toLowerCase()
+                .trim();
+            if (key && seen_keys.has(key)) {
+                warnings.push(
+                    `Row ${index + 1}: Duplicate "${dedup_field}" value "${key}" (same as row ${seen_keys.get(key) + 1})`,
+                );
+            } else if (key) {
+                seen_keys.set(key, index);
+            }
+        });
+        return warnings;
+    }
 }

src/app/overlays/bulk-item-modal/csv-upload.component.ts

@@ -19,12 +19,15 @@ import { TranslatePipe } from '../../ui/translate.pipe';
                 (dragenter)="dragging = true"
                 (dragleave)="dragging = false"
                 (dragend)="dragging = false"
+                (dragover)="onDragOver($event)"
+                (drop)="onDrop($event)"
             >
                 <icon class="text-6xl">cloud_upload</icon>
                 <div class="text">{{ 'COMMON.BULK_DROP_MSG' | translate }}</div>
                 <input
                     class="absolute inset-0 opacity-0"
                     type="file"
+                    accept=".csv,.tsv"
                     (change)="loadCSVData($event)"
                 />
             </button>
@@ -69,31 +72,50 @@ export class CsvUploadComponent {
     /** Whether CSV data is being processed */
     public loading: boolean;
 
+    public onDragOver(event: DragEvent) {
+        event.preventDefault();
+        event.stopPropagation();
+    }
+
+    public onDrop(event: DragEvent) {
+        event.preventDefault();
+        event.stopPropagation();
+        this.dragging = false;
+        const file = event.dataTransfer?.files?.[0];
+        if (file) {
+            this.loadFile(file);
+        }
+    }
+
     public loadCSVData(event) {
-        this.loading = true;
         /* istanbul ignore else */
         if (event.target) {
             const element = event.target as HTMLInputElement;
             const file = element.files[0];
             /* istanbul ignore else */
             if (file) {
-                const reader = new FileReader();
-                reader.readAsText(file, 'UTF-8');
-                reader.addEventListener('load', (evt) => {
-                    this.processCSVData(
-                        (evt.target as FileReader).result as string,
-                        file.name.endsWith('.csv') ? ',' : '\t',
-                    );
-                    element.value = '';
-                });
-                reader.addEventListener('error', (_) => {
-                    this.loading = false;
-                    notifyError('Error reading file.');
-                });
+                this.loadFile(file);
+                element.value = '';
             }
         }
     }
 
+    private loadFile(file: File) {
+        this.loading = true;
+        const reader = new FileReader();
+        reader.readAsText(file, 'UTF-8');
+        reader.addEventListener('load', (evt) => {
+            this.processCSVData(
+                (evt.target as FileReader).result as string,
+                file.name.endsWith('.csv') ? ',' : '\t',
+            );
+        });
+        reader.addEventListener('error', (_) => {
+            this.loading = false;
+            notifyError('Error reading file.');
+        });
+    }
+
     public downloadTemplateCSV() {
         const ignore_keys = ['module_list', 'settings', '_type', 'version'];
         const csv_data = jsonToCsv(
@@ -112,7 +134,11 @@ export class CsvUploadComponent {
             this.loading = false;
             this.list.emit(list);
         } catch (e) {
+            this.loading = false;
             console.error(e);
+            notifyError(
+                'Error parsing CSV data. Please check the file format.',
+            );
         }
     }
 }

src/app/overlays/bulk-item-modal/status-list.component.ts

@@ -4,22 +4,37 @@ import {
     SimpleChanges,
     input,
     output,
+    signal,
 } from '@angular/core';
 import { MatProgressSpinnerModule } from '@angular/material/progress-spinner';
 import { MatTooltipModule } from '@angular/material/tooltip';
 import { PlaceResource } from '@placeos/ts-client';
-import { Observable } from 'rxjs';
+import { Observable, firstValueFrom } from 'rxjs';
+import { notifyError } from '../../common/notifications';
 import { IconComponent } from '../../ui/icon.component';
 import { TranslatePipe } from '../../ui/translate.pipe';
 
+const BATCH_SIZE = 5;
+
 @Component({
     selector: 'bulk-item-status-list',
     template: `
         <div class="flex flex-col items-center px-4 pb-4">
-            @if (!done) {
+            @if (!is_done()) {
                 <div class="info">
                     {{ 'COMMON.BULK_UPLOADING' | translate }}
                 </div>
+                <div class="text-base-content/60 mb-2 text-sm">
+                    {{ completed_count() }} / {{ list().length }} completed
+                    @if (error_count() > 0) {
+                        ({{ error_count() }} failed)
+                    }
+                </div>
+            } @else {
+                <div class="mb-2 text-sm">
+                    {{ completed_count() }} succeeded, {{ error_count() }}
+                    failed
+                </div>
             }
             @for (item of list(); track item.id; let i = $index) {
                 <div
@@ -34,7 +49,7 @@ import { TranslatePipe } from '../../ui/translate.pipe';
                         }
                     </div>
                     <div class="status">
-                        @if (status[i] !== 'loading') {
+                        @if (status[i] && status[i] !== 'loading') {
                             <div
                                 class="flex h-8 w-8 items-center justify-center rounded-full text-2xl shadow-sm"
                                 [class.bg-error]="status[i] !== 'done'"
@@ -82,6 +97,12 @@ export class StatusListComponent implements OnChanges {
     public readonly done = output<Record<string, unknown>[]>();
     /** Status of each of the items to be created */
     public status: Record<string, string> = {};
+    /** Whether all items have been processed */
+    public readonly is_done = signal(false);
+    /** Count of successfully completed items */
+    public readonly completed_count = signal(0);
+    /** Count of failed items */
+    public readonly error_count = signal(0);
 
     public ngOnChanges(changes: SimpleChanges) {
         if (changes.list && this.list()) {
@@ -91,32 +112,69 @@ export class StatusListComponent implements OnChanges {
 
     public async saveItems() {
         try {
-            const list = [];
-            let index = 0;
-            for (const item of this.list()) {
-                this.status[index] = 'loading';
-                const saved_item = await this.save()({ ...item, id: '' })
-                    .toPromise()
-                    .catch((err) => {
-                        console.log('Error:', err);
-                        this.status[index] = `Error: ${err.status || err} ${
-                            err.statusText || err
-                        }`;
-                        console.error(this.status[index]);
-                        // notifyError(this.status[index]);
-                    });
-                list.push(saved_item);
-                if (this.status[index] === 'loading') {
-                    this.status[index] = 'done';
-                }
-                index++;
+            const items = this.list();
+            const results: (PlaceResource | undefined)[] = new Array(
+                items.length,
+            );
+            let success_count = 0;
+            let fail_count = 0;
+
+            for (let i = 0; i < items.length; i += BATCH_SIZE) {
+                const batch = items.slice(i, i + BATCH_SIZE);
+                const batch_promises = batch.map(async (item, batch_index) => {
+                    const index = i + batch_index;
+                    this.status[index] = 'loading';
+                    try {
+                        const saved_item = await firstValueFrom(
+                            this.save()({ ...item, id: '' }),
+                        );
+                        this.status[index] = 'done';
+                        success_count++;
+                        this.completed_count.set(success_count);
+                        results[index] = saved_item;
+                    } catch (err) {
+                        const message = this.formatError(err);
+                        this.status[index] = message;
+                        console.error(`Failed to save item ${index}:`, err);
+                        notifyError(message);
+                        fail_count++;
+                        this.error_count.set(fail_count);
+                    }
+                });
+                await Promise.all(batch_promises);
             }
-            const clean_list = list.filter((item) => !!item);
+
+            this.is_done.set(true);
+            const clean_list = results.filter((item) => !!item);
             if (clean_list.length > 0) {
-                this.done.emit(clean_list);
+                this.done.emit(
+                    clean_list as unknown as Record<string, unknown>[],
+                );
             }
         } catch (e) {
             console.error(e);
         }
     }
+
+    private formatError(err: unknown): string {
+        if (err && typeof err === 'object') {
+            const http_err = err as Record<string, unknown>;
+            const status = http_err.status || '';
+            const status_text = http_err.statusText || '';
+            const message =
+                http_err.message ||
+                (http_err.error &&
+                typeof http_err.error === 'object' &&
+                (http_err.error as Record<string, unknown>).message
+                    ? (http_err.error as Record<string, unknown>).message
+                    : '');
+            if (status || status_text) {
+                return `Error: ${status} ${status_text}`.trim();
+            }
+            if (message) {
+                return `Error: ${message}`;
+            }
+        }
+        return `Error: ${String(err)}`;
+    }
 }