From b51f7ee7e6960b82893764b61278a9dcbdf1c76d Mon Sep 17 00:00:00 2001 From: Alexander Ng Date: Mon, 2 Mar 2026 09:38:07 -0800 Subject: [PATCH 1/4] refactor: migrate CSV parser to async implementation with progress tracking - Replaced synchronous `parseCSVToPVs` with async `parseCSVToPVsAsync` for better performance with large files - Added progress tracking with `ParseProgress` type to show parsing and validation status - Implemented chunked validation processing (50 rows per chunk) to prevent main thread blocking - Added progress indicators for both parsing and validation phases - Updated CSVImportDialog to handle async operations with proper state management - Enhanced error handling and user feedback during import process --- src/components/CSVImportDialog.tsx | 103 +++++++++++++++++++--- src/utils/csvParser.ts | 137 +++++++++++++++++------------ 2 files changed, 171 insertions(+), 69 deletions(-) diff --git a/src/components/CSVImportDialog.tsx b/src/components/CSVImportDialog.tsx index ae96e8c..118b9d8 100644 --- a/src/components/CSVImportDialog.tsx +++ b/src/components/CSVImportDialog.tsx @@ -21,10 +21,11 @@ import { } from '@mui/material'; import { Upload } from '@mui/icons-material'; import { - parseCSVToPVs, + parseCSVToPVsAsync, createTagMapping, createValidationSummary, ParsedCSVRow, + ParseProgress, } from '../utils/csvParser'; interface CSVImportDialogProps { @@ -50,6 +51,8 @@ export function CSVImportDialog({ const [importing, setImporting] = useState(false); const [fileSelected, setFileSelected] = useState(false); const [importError, setImportError] = useState(null); + const [parsingProgress, setParsingProgress] = useState(null); + const [parsing, setParsing] = useState(false); const handleClose = () => { setCSVData([]); @@ -67,39 +70,85 @@ export function CSVImportDialog({ if (!file) return; try { + setParsing(true); + setParsingProgress({ processedRows: 0, totalRows: 0, status: 'parsing' }); + setParseErrors([]); + setCSVData([]); + setValidationSummary(''); + setFileSelected(false); + const content = await file.text(); - const result = parseCSVToPVs(content); + + // Use async parser for better performance with large files + const result = await parseCSVToPVsAsync(content, (progress) => { + setParsingProgress(progress); + }); if (result.errors.length > 0) { setParseErrors(result.errors); setCSVData([]); setValidationSummary(''); setFileSelected(false); + setParsing(false); + setParsingProgress(null); return; } setCSVData(result.data); setParseErrors([]); setFileSelected(true); + setParsing(false); + setParsingProgress(null); - // Validate tags + // Validate tags with progress feedback if (result.data.length > 0) { - // Collect all rejected groups and values across all rows + setParsing(true); + setParsingProgress({ + processedRows: 0, + totalRows: result.data.length, + status: 'validating', + }); + + // PERFORMANCE: Collect all rejected groups and values across all rows + // This validation ensures that only valid tag groups and values are imported const allRejectedGroups = new Set(); const allRejectedValues: Record> = {}; - result.data.forEach((row) => { - const mapping = createTagMapping(row.groups, availableTagGroups); + // PERFORMANCE: Process validation in chunks to prevent blocking the main thread + // Chunk size of 50 provides good balance between performance and responsiveness + const chunkSize = 50; + for (let start = 0; start < result.data.length; start += chunkSize) { + const chunk = result.data.slice(start, start + chunkSize); + + // PERFORMANCE: Process each row in the current chunk + chunk.forEach((row) => { + const mapping = createTagMapping(row.groups, availableTagGroups); - mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); + // Collect rejected groups (groups that don't exist in backend) + mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); - Object.entries(mapping.rejectedValues).forEach(([group, values]) => { - if (!allRejectedValues[group]) { - allRejectedValues[group] = new Set(); - } - values.forEach((value) => allRejectedValues[group].add(value)); + // Collect rejected values (values that don't exist for their group in backend) + Object.entries(mapping.rejectedValues).forEach(([group, values]) => { + if (!allRejectedValues[group]) { + allRejectedValues[group] = new Set(); + } + values.forEach((value) => allRejectedValues[group].add(value)); + }); }); - }); + + // PERFORMANCE: Update progress indicator for real-time UI feedback + const processedRows = Math.min(start + chunk.length, result.data.length); + setParsingProgress({ + processedRows, + totalRows: result.data.length, + status: 'validating', + }); + + // PERFORMANCE: Yield control to allow UI updates and prevent blocking + // This is crucial for maintaining responsive UI during validation of large datasets + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await new Promise((resolve) => setTimeout(resolve, 0)); + } // Convert sets to arrays const rejectedGroups = Array.from(allRejectedGroups); @@ -110,6 +159,8 @@ export function CSVImportDialog({ const summary = createValidationSummary(rejectedGroups, rejectedValues); setValidationSummary(summary); + setParsing(false); + setParsingProgress(null); } } catch (error) { setParseErrors([ @@ -118,6 +169,8 @@ export function CSVImportDialog({ setCSVData([]); setValidationSummary(''); setFileSelected(false); + setParsing(false); + setParsingProgress(null); } // Reset file input @@ -158,13 +211,35 @@ export function CSVImportDialog({ variant="contained" component="span" startIcon={} - disabled={importing} + disabled={importing || parsing} > Select CSV File + {/* Parsing Progress */} + {parsing && parsingProgress && ( + + + + + + {parsingProgress.status === 'parsing' + ? 'Parsing CSV file...' + : 'Validating tags...'} + + + Processed {parsingProgress.processedRows} of {parsingProgress.totalRows} rows + + + + {Math.round((parsingProgress.processedRows / parsingProgress.totalRows) * 100)}% + + + + )} + {/* CSV Format Instructions */} diff --git a/src/utils/csvParser.ts b/src/utils/csvParser.ts index cdcb8e4..8f7de22 100644 --- a/src/utils/csvParser.ts +++ b/src/utils/csvParser.ts @@ -19,6 +19,12 @@ export interface ParsedCSVResult { errors: string[]; } +export interface ParseProgress { + processedRows: number; + totalRows: number; + status: 'parsing' | 'validating' | 'complete'; +} + /** * Parse a single CSV line, handling quoted fields * Simple CSV parser that handles basic quoting @@ -57,19 +63,17 @@ function parseCSVLine(line: string): string[] { } /** - * Parse CSV file content into PV data structure - * - * CSV Format (matches Python parse_csv_to_dict): - * - Required columns: "Setpoint" or "Readback" (at least one) - * - Optional columns: "Device", "Description" - * - Any additional columns are treated as tag groups - * - Tag values can be comma-separated (e.g., "tag1, tag2") - * - Filters out 'nan' and 'none' values + * Async CSV parser with progress feedback for large files + * Processes the file in chunks to prevent UI blocking and provides progress updates * * @param csvContent - Raw CSV file content as string + * @param onProgress - Optional callback for progress updates (processedRows, totalRows, status) * @returns Parsed PV data with tag groups and any errors */ -export function parseCSVToPVs(csvContent: string): ParsedCSVResult { +export async function parseCSVToPVsAsync( + csvContent: string, + onProgress?: (progress: ParseProgress) => void +): Promise { const errors: string[] = []; const data: ParsedCSVRow[] = []; @@ -100,58 +104,81 @@ export function parseCSVToPVs(csvContent: string): ParsedCSVResult { const standardColumns = ['Setpoint', 'Readback', 'Device', 'Description']; const groupColumns = cleanedHeaders.filter((col) => !standardColumns.includes(col)); - // Parse data rows (starting from row 2 in 1-indexed terms, row 1 in 0-indexed) - for (let i = 1; i < lines.length; i += 1) { - const line = lines[i].trim(); - if (line) { - const rowValues = parseCSVLine(line); + // Process data rows in chunks to prevent blocking + const totalRows = lines.length - 1; // Exclude header + const chunkSize = 100; // Process 100 rows at a time - // Create a row dictionary - const rowDict: Record = {}; - cleanedHeaders.forEach((header, index) => { - rowDict[header] = index < rowValues.length ? rowValues[index].trim() : ''; - }); + for (let start = 1; start < lines.length; start += chunkSize) { + const chunkEnd = Math.min(start + chunkSize, lines.length); + const chunkLines = lines.slice(start, chunkEnd); - const setpoint = rowDict.Setpoint || ''; - const readback = rowDict.Readback || ''; - - // Only process row if at least one of setpoint or readback is present - if (setpoint || readback) { - const device = rowDict.Device || ''; - const description = rowDict.Description || ''; - - // Parse tag groups - const groups: Record = {}; - - groupColumns.forEach((groupName) => { - const cellValue = rowDict[groupName] || ''; - const trimmedValue = cellValue.trim(); - - if ( - trimmedValue && - trimmedValue.toLowerCase() !== 'nan' && - trimmedValue.toLowerCase() !== 'none' - ) { - // Split comma-separated values and filter - const tagValues = trimmedValue - .split(',') - .map((val) => val.trim()) - .filter((val) => val); - groups[groupName] = tagValues; - } else { - groups[groupName] = []; - } - }); + // Process chunk + for (let i = 0; i < chunkLines.length; i += 1) { + const line = chunkLines[i].trim(); + if (line) { + const rowValues = parseCSVLine(line); - data.push({ - Setpoint: setpoint, - Readback: readback, - Device: device, - Description: description, - groups, + // Create a row dictionary + const rowDict: Record = {}; + cleanedHeaders.forEach((header, index) => { + rowDict[header] = index < rowValues.length ? rowValues[index].trim() : ''; }); + + const setpoint = rowDict.Setpoint || ''; + const readback = rowDict.Readback || ''; + + // Only process row if at least one of setpoint or readback is present + if (setpoint || readback) { + const device = rowDict.Device || ''; + const description = rowDict.Description || ''; + + // Parse tag groups + const groups: Record = {}; + + groupColumns.forEach((groupName) => { + const cellValue = rowDict[groupName] || ''; + const trimmedValue = cellValue.trim(); + + if ( + trimmedValue && + trimmedValue.toLowerCase() !== 'nan' && + trimmedValue.toLowerCase() !== 'none' + ) { + // Split comma-separated values and filter + const tagValues = trimmedValue + .split(',') + .map((val) => val.trim()) + .filter((val) => val); + groups[groupName] = tagValues; + } else { + groups[groupName] = []; + } + }); + + data.push({ + Setpoint: setpoint, + Readback: readback, + Device: device, + Description: description, + groups, + }); + } } } + + // Report progress and yield control to prevent blocking + const processedRows = Math.min(start + chunkLines.length - 1, totalRows); + if (onProgress) { + onProgress({ + processedRows, + totalRows, + status: 'parsing', + }); + } + + // Yield control to allow UI updates + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await new Promise((resolve) => setTimeout(resolve, 0)); } return { From e1d04d9115a05ab2f8ad7035088989acb349027b Mon Sep 17 00:00:00 2001 From: Alexander Ng Date: Mon, 2 Mar 2026 10:37:28 -0800 Subject: [PATCH 2/4] feat: implement bulk tag validation for CSV import with progress feedback --- src/components/CSVImportDialog.tsx | 63 ++++++------------------- src/utils/csvParser.ts | 74 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/src/components/CSVImportDialog.tsx b/src/components/CSVImportDialog.tsx index 118b9d8..ffeef4c 100644 --- a/src/components/CSVImportDialog.tsx +++ b/src/components/CSVImportDialog.tsx @@ -22,8 +22,8 @@ import { import { Upload } from '@mui/icons-material'; import { parseCSVToPVsAsync, - createTagMapping, createValidationSummary, + validateCSVTags, ParsedCSVRow, ParseProgress, } from '../utils/csvParser'; @@ -109,55 +109,22 @@ export function CSVImportDialog({ status: 'validating', }); - // PERFORMANCE: Collect all rejected groups and values across all rows - // This validation ensures that only valid tag groups and values are imported - const allRejectedGroups = new Set(); - const allRejectedValues: Record> = {}; - - // PERFORMANCE: Process validation in chunks to prevent blocking the main thread - // Chunk size of 50 provides good balance between performance and responsiveness - const chunkSize = 50; - for (let start = 0; start < result.data.length; start += chunkSize) { - const chunk = result.data.slice(start, start + chunkSize); - - // PERFORMANCE: Process each row in the current chunk - chunk.forEach((row) => { - const mapping = createTagMapping(row.groups, availableTagGroups); - - // Collect rejected groups (groups that don't exist in backend) - mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); - - // Collect rejected values (values that don't exist for their group in backend) - Object.entries(mapping.rejectedValues).forEach(([group, values]) => { - if (!allRejectedValues[group]) { - allRejectedValues[group] = new Set(); - } - values.forEach((value) => allRejectedValues[group].add(value)); + const validationResults = await validateCSVTags( + result.data, + availableTagGroups, + (processedRows, totalRows) => { + setParsingProgress({ + processedRows, + totalRows, + status: 'validating', }); - }); - - // PERFORMANCE: Update progress indicator for real-time UI feedback - const processedRows = Math.min(start + chunk.length, result.data.length); - setParsingProgress({ - processedRows, - totalRows: result.data.length, - status: 'validating', - }); - - // PERFORMANCE: Yield control to allow UI updates and prevent blocking - // This is crucial for maintaining responsive UI during validation of large datasets - // eslint-disable-next-line no-await-in-loop, no-promise-executor-return - await new Promise((resolve) => setTimeout(resolve, 0)); - } - - // Convert sets to arrays - const rejectedGroups = Array.from(allRejectedGroups); - const rejectedValues: Record = {}; - Object.entries(allRejectedValues).forEach(([group, valueSet]) => { - rejectedValues[group] = Array.from(valueSet); - }); + } + ); - const summary = createValidationSummary(rejectedGroups, rejectedValues); + const summary = createValidationSummary( + validationResults.rejectedGroups, + validationResults.rejectedValues + ); setValidationSummary(summary); setParsing(false); setParsingProgress(null); diff --git a/src/utils/csvParser.ts b/src/utils/csvParser.ts index 8f7de22..fd1ead3 100644 --- a/src/utils/csvParser.ts +++ b/src/utils/csvParser.ts @@ -279,3 +279,77 @@ export function createValidationSummary( return summaryParts.length > 0 ? summaryParts.join(' • ') : 'All groups and values are valid'; } + +/** + * Bulk tag validation for CSV import + * Validates all CSV rows against available tag groups and provides progress feedback + * + * @param csvData - Array of parsed CSV rows with tag groups + * @param availableTagGroups - Available tag groups from backend + * @param onProgress - Optional callback for progress updates (processedRows, totalRows) + * @returns Validation results with rejected groups and values + */ +export interface BulkTagValidationResult { + rejectedGroups: string[]; + rejectedValues: Record; +} + +export async function validateCSVTags( + csvData: ParsedCSVRow[], + availableTagGroups: Array<{ + id: string; + name: string; + tags: Array<{ id: string; name: string }>; + }>, + onProgress?: (processedRows: number, totalRows: number) => void +): Promise { + // Collect all rejected groups and values across all rows + const allRejectedGroups = new Set(); + const allRejectedValues: Record> = {}; + + // Process validation in chunks to prevent blocking the main thread + const chunkSize = 50; + const totalRows = csvData.length; + + for (let start = 0; start < csvData.length; start += chunkSize) { + const chunk = csvData.slice(start, start + chunkSize); + + // Process each row in the current chunk + chunk.forEach((row) => { + const mapping = createTagMapping(row.groups, availableTagGroups); + + // Collect rejected groups (groups that don't exist in backend) + mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); + + // Collect rejected values (values that don't exist for their group in backend) + Object.entries(mapping.rejectedValues).forEach(([group, values]) => { + if (!allRejectedValues[group]) { + allRejectedValues[group] = new Set(); + } + values.forEach((value) => allRejectedValues[group].add(value)); + }); + }); + + // Update progress indicator for real-time UI feedback + const processedRows = Math.min(start + chunk.length, totalRows); + if (onProgress) { + onProgress(processedRows, totalRows); + } + + // Yield control to allow UI updates and prevent blocking + // eslint-disable-next-line no-await-in-loop, no-promise-executor-return + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + // Convert sets to arrays + const rejectedGroups = Array.from(allRejectedGroups); + const rejectedValues: Record = {}; + Object.entries(allRejectedValues).forEach(([group, valueSet]) => { + rejectedValues[group] = Array.from(valueSet); + }); + + return { + rejectedGroups, + rejectedValues, + }; +} From d6116fb8114fd0fd405fb194ded1ac2d67f69c72 Mon Sep 17 00:00:00 2001 From: Alexander Ng Date: Mon, 2 Mar 2026 11:54:00 -0800 Subject: [PATCH 3/4] refactor: simplify CSV tag validation by removing async progress tracking Removed the async progress callback and chunked processing from `validateCSVTags` function. The validation now runs synchronously without progress updates, simplifying the implementation while maintaining the same validation logic and results. Updated function signature and removed progress-related code from both the validation function and its caller in CSVImportDialog. --- src/components/CSVImportDialog.tsx | 12 +------ src/utils/csvParser.ts | 52 +++++++++--------------------- 2 files changed, 17 insertions(+), 47 deletions(-) diff --git a/src/components/CSVImportDialog.tsx b/src/components/CSVImportDialog.tsx index ffeef4c..6a182f8 100644 --- a/src/components/CSVImportDialog.tsx +++ b/src/components/CSVImportDialog.tsx @@ -109,17 +109,7 @@ export function CSVImportDialog({ status: 'validating', }); - const validationResults = await validateCSVTags( - result.data, - availableTagGroups, - (processedRows, totalRows) => { - setParsingProgress({ - processedRows, - totalRows, - status: 'validating', - }); - } - ); + const validationResults = await validateCSVTags(result.data, availableTagGroups); const summary = createValidationSummary( validationResults.rejectedGroups, diff --git a/src/utils/csvParser.ts b/src/utils/csvParser.ts index fd1ead3..5493c16 100644 --- a/src/utils/csvParser.ts +++ b/src/utils/csvParser.ts @@ -282,11 +282,10 @@ export function createValidationSummary( /** * Bulk tag validation for CSV import - * Validates all CSV rows against available tag groups and provides progress feedback + * Validates all CSV rows against available tag groups * * @param csvData - Array of parsed CSV rows with tag groups * @param availableTagGroups - Available tag groups from backend - * @param onProgress - Optional callback for progress updates (processedRows, totalRows) * @returns Validation results with rejected groups and values */ export interface BulkTagValidationResult { @@ -294,52 +293,33 @@ export interface BulkTagValidationResult { rejectedValues: Record; } -export async function validateCSVTags( +export function validateCSVTags( csvData: ParsedCSVRow[], availableTagGroups: Array<{ id: string; name: string; tags: Array<{ id: string; name: string }>; - }>, - onProgress?: (processedRows: number, totalRows: number) => void -): Promise { + }> +): BulkTagValidationResult { // Collect all rejected groups and values across all rows const allRejectedGroups = new Set(); const allRejectedValues: Record> = {}; - // Process validation in chunks to prevent blocking the main thread - const chunkSize = 50; - const totalRows = csvData.length; + // Process each row + csvData.forEach((row) => { + const mapping = createTagMapping(row.groups, availableTagGroups); - for (let start = 0; start < csvData.length; start += chunkSize) { - const chunk = csvData.slice(start, start + chunkSize); + // Collect rejected groups (groups that don't exist in backend) + mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); - // Process each row in the current chunk - chunk.forEach((row) => { - const mapping = createTagMapping(row.groups, availableTagGroups); - - // Collect rejected groups (groups that don't exist in backend) - mapping.rejectedGroups.forEach((group) => allRejectedGroups.add(group)); - - // Collect rejected values (values that don't exist for their group in backend) - Object.entries(mapping.rejectedValues).forEach(([group, values]) => { - if (!allRejectedValues[group]) { - allRejectedValues[group] = new Set(); - } - values.forEach((value) => allRejectedValues[group].add(value)); - }); + // Collect rejected values (values that don't exist for their group in backend) + Object.entries(mapping.rejectedValues).forEach(([group, values]) => { + if (!allRejectedValues[group]) { + allRejectedValues[group] = new Set(); + } + values.forEach((value) => allRejectedValues[group].add(value)); }); - - // Update progress indicator for real-time UI feedback - const processedRows = Math.min(start + chunk.length, totalRows); - if (onProgress) { - onProgress(processedRows, totalRows); - } - - // Yield control to allow UI updates and prevent blocking - // eslint-disable-next-line no-await-in-loop, no-promise-executor-return - await new Promise((resolve) => setTimeout(resolve, 0)); - } + }); // Convert sets to arrays const rejectedGroups = Array.from(allRejectedGroups); From 06fac089789d130da0ae4f8f9ebac0af465840e7 Mon Sep 17 00:00:00 2001 From: Devan Agrawal Date: Mon, 30 Mar 2026 10:16:10 -0700 Subject: [PATCH 4/4] remove deprecated shebang in husky file --- .husky/pre-commit | 2 -- 1 file changed, 2 deletions(-) diff --git a/.husky/pre-commit b/.husky/pre-commit index 81a3ef5..6e47593 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,5 +1,3 @@ -#!/usr/bin/env sh - # Run TypeScript type checking pnpm exec tsc --noEmit