Merge branch 'main' of https://github.com/willwade/AACProcessors-nodejs

Author: willwade

src/core/fileProcessor.ts

@@ -26,6 +26,13 @@ class FileProcessor {
   static detectFormat(filePathOrBuffer: string | Buffer): FileFormat {
     if (typeof filePathOrBuffer === 'string') {
       const ext = path.extname(filePathOrBuffer).toLowerCase();
+      const fileName = path.basename(filePathOrBuffer).toLowerCase();
+
+      // Handle double extensions like .sub.zip
+      if (fileName.endsWith('.sub.zip') || ext === '.sub') {
+        return 'snap';
+      }
+
       switch (ext) {
         case '.gridset':
         case '.gridsetx':

src/processors/obfProcessor.ts

@@ -51,6 +51,16 @@ interface ObfButton {
   image_id?: string; // Reference to image in the images array
 }
 
+interface ObfManifest {
+  format?: string;
+  root?: string;
+  paths?: {
+    boards?: { [key: string]: string };
+    images?: { [key: string]: string };
+    sounds?: { [key: string]: string };
+  };
+}
+
 /**
  * Map OBF hidden value to AAC standard visibility
  * OBF: true = hidden, false/undefined = visible
@@ -201,15 +211,14 @@ class ObfProcessor extends BaseProcessor {
     }
   }
 
-  private async processBoard(boardData: ObfBoard, _boardPath: string): Promise<AACPage> {
+  private async processBoard(
+    boardData: ObfBoard,
+    _boardPath: string,
+    isZipEntry: boolean
+  ): Promise<AACPage> {
     const sourceButtons = boardData.buttons || [];
 
     // Calculate page ID first (used to make button IDs unique)
-    const isZipEntry =
-      _boardPath &&
-      _boardPath.endsWith('.obf') &&
-      !_boardPath.includes('/') &&
-      !_boardPath.includes('\\');
     const pageId = isZipEntry
       ? _boardPath // Zip entry - use filename to match navigation paths
       : boardData?.id
@@ -437,7 +446,7 @@ class ObfProcessor extends BaseProcessor {
         const boardData = tryParseObfJson(content);
         if (boardData) {
           console.log('[OBF] Detected .obf file, parsed as JSON');
-          const page = await this.processBoard(boardData, filePathOrBuffer);
+          const page = await this.processBoard(boardData, filePathOrBuffer, false);
           tree.addPage(page);
 
           // Set metadata from root board
@@ -475,7 +484,7 @@ class ObfProcessor extends BaseProcessor {
       const asJson = tryParseObfJson(filePathOrBuffer);
       if (!asJson) throw new Error('Invalid OBF content: not JSON and not ZIP');
       console.log('[OBF] Detected buffer/string as OBF JSON');
-      const page = await this.processBoard(asJson, '[bufferOrString]');
+      const page = await this.processBoard(asJson, '[bufferOrString]', false);
       tree.addPage(page);
 
       // Set metadata from root board
@@ -508,18 +517,43 @@ class ObfProcessor extends BaseProcessor {
 
     console.log('[OBF] Detected zip archive, extracting .obf files');
 
-    // Collect all .obf entries
-    const obfEntries = this.zipFile
-      .listFiles()
-      .filter((name) => name.toLowerCase().endsWith('.obf'));
+    // List manifest and OBF files
+    const filesInZip = this.zipFile.listFiles();
+    const manifestFile = filesInZip.filter((name) => name.toLowerCase() === 'manifest.json');
+    let obfEntries = filesInZip.filter((name) => name.toLowerCase().endsWith('.obf'));
+
+    // Attempt to read manifest
+    if (manifestFile && manifestFile.length === 1) {
+      try {
+        const content = await this.zipFile.readFile(manifestFile[0]);
+        const data = decodeText(content);
+        const str = typeof data === 'string' ? data : readTextFromInput(data);
+        if (!str.trim()) throw new Error('Manifest object missing');
+        const manifestObject = JSON.parse(str) as ObfManifest;
+        if (!manifestObject) throw new Error('Manifest object is empty');
+
+        // Replace OBF file list
+        if (manifestObject.paths && manifestObject.paths.boards) {
+          obfEntries = Object.values(manifestObject.paths.boards);
+        }
+
+        // Move root board to top of list
+        if (manifestObject.root) {
+          obfEntries = obfEntries.filter((item) => item !== manifestObject.root);
+          obfEntries.unshift(manifestObject.root);
+        }
+      } catch (err) {
+        console.warn('[OBF] Error processing mainfest', err);
+      }
+    }
 
     // Process each .obf entry
     for (const entryName of obfEntries) {
       try {
         const content = await this.zipFile.readFile(entryName);
         const boardData = tryParseObfJson(decodeText(content));
         if (boardData) {
-          const page = await this.processBoard(boardData, entryName);
+          const page = await this.processBoard(boardData, entryName, true);
           tree.addPage(page);
 
           // Set metadata if not already set (use first board as reference)

src/processors/snapProcessor.ts

@@ -17,8 +17,9 @@ import {
 import { generateCloneId } from '../utilities/analytics/utils/idGenerator';
 import { SnapValidator } from '../validation/snapValidator';
 import { ValidationResult } from '../validation/validationTypes';
-import { ProcessorInput, getFs, getNodeRequire, getPath, isNodeRuntime } from '../utils/io';
+import { ProcessorInput, getFs, getNodeRequire, getPath, isNodeRuntime, getOs } from '../utils/io';
 import { openSqliteDatabase, requireBetterSqlite3 } from '../utils/sqlite';
+import { openZipFromInput } from '../utils/zip';
 
 /**
  * Convert a Buffer or Uint8Array to base64 string (browser and Node compatible)
@@ -123,8 +124,45 @@ class SnapProcessor extends BaseProcessor {
     await Promise.resolve();
     const tree = new AACTree();
     let dbResult: Awaited<ReturnType<typeof openSqliteDatabase>> | null = null;
+    let cleanupTempZip: (() => void) | null = null;
+
     try {
-      dbResult = await openSqliteDatabase(filePathOrBuffer, { readonly: true });
+      // Handle .sub.zip files (Snap pageset backups containing .sps files)
+      let inputFile = filePathOrBuffer;
+
+      if (typeof filePathOrBuffer === 'string') {
+        const fileName = getPath().basename(filePathOrBuffer).toLowerCase();
+        if (fileName.endsWith('.sub.zip') || filePathOrBuffer.endsWith('.sub')) {
+          const fs = getFs();
+          const path = getPath();
+          const os = getOs();
+
+          // Extract .sub.zip to find the embedded .sps file
+          const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'snap-sub-'));
+          const { zip } = await openZipFromInput(filePathOrBuffer);
+
+          // Find the .sps file in the archive
+          const files = zip.listFiles();
+          const spsFile = files.find((f) => f.endsWith('.sps'));
+
+          if (!spsFile) {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+            throw new Error('No .sps file found in .sub.zip archive');
+          }
+
+          // Extract the .sps file
+          const spsData = await zip.readFile(spsFile);
+          const extractedSpsPath = path.join(tempDir, path.basename(spsFile));
+          fs.writeFileSync(extractedSpsPath, Buffer.from(spsData));
+
+          inputFile = extractedSpsPath;
+          cleanupTempZip = () => {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+          };
+        }
+      }
+
+      dbResult = await openSqliteDatabase(inputFile, { readonly: true });
       const db = dbResult.db;
 
       const getTableColumns = (tableName: string): Set<string> => {
@@ -789,6 +827,14 @@ class SnapProcessor extends BaseProcessor {
       } else if (dbResult?.db) {
         dbResult.db.close();
       }
+      // Clean up temporary extracted .sps file from .sub.zip
+      if (cleanupTempZip) {
+        try {
+          cleanupTempZip();
+        } catch (e) {
+          console.warn('[SnapProcessor] Failed to clean up temporary .sps file:', e);
+        }
+      }
     }
   }
 

test/assets/snap/example.sub.zip

@@ -125,6 +125,9 @@ describe('FileProcessor', () => {
       it('should detect snap format', async () => {
         expect(FileProcessor.detectFormat('test.sps')).toBe('snap');
         expect(FileProcessor.detectFormat('test.spb')).toBe('snap');
+        expect(FileProcessor.detectFormat('test.sub')).toBe('snap');
+        expect(FileProcessor.detectFormat('test.sub.zip')).toBe('snap');
+        expect(FileProcessor.detectFormat('/path/to/My Pageset.sub.zip')).toBe('snap');
       });
 
       it('should detect dot format', async () => {

test/core/fileProcessor.test.ts

@@ -1,10 +1,13 @@
 import { SnapProcessor } from '../src/processors/snapProcessor';
 import { AACTree } from '../src/core/treeStructure';
 import path from 'path';
+import fs from 'fs';
+import AdmZip from 'adm-zip';
 
 describe('SnapProcessor', () => {
   const exampleFile: string = path.join(__dirname, 'assets/snap/example.spb');
   const exampleSPSFile: string = path.join(__dirname, 'assets/snap/example.sps');
+  const exampleSubZipFile: string = path.join(__dirname, 'assets/snap/example.sub.zip');
 
   it('should extract all texts from a .spb file', async () => {
     const processor = new SnapProcessor();
@@ -58,6 +61,14 @@ describe('SnapProcessor', () => {
     }
   });
 
+  it('should handle .sub.zip files by extracting and processing the embedded .sps file', async () => {
+    const processor = new SnapProcessor();
+    const tree: AACTree = await processor.loadIntoTree(exampleSubZipFile);
+    expect(tree).toBeTruthy();
+    const pageIds: string[] = Object.keys(tree.pages);
+    expect(pageIds.length).toBeGreaterThan(0);
+  });
+
   describe('Error Handling', () => {
     it('should throw error for non-existent file', async () => {
       const processor = new SnapProcessor();
@@ -74,6 +85,26 @@ describe('SnapProcessor', () => {
       const processor = new SnapProcessor();
       await expect(processor.loadIntoTree('')).rejects.toThrow();
     });
+
+    it('should throw error for .sub.zip file without .sps file inside', async () => {
+      const processor = new SnapProcessor();
+      const zip = new AdmZip();
+      zip.addFile('not-an-sps.txt', Buffer.from('Not a pageset'));
+      const invalidSubZipPath = path.join(__dirname, 'invalid.sub.zip');
+      zip.writeZip(invalidSubZipPath);
+
+      await expect(processor.loadIntoTree(invalidSubZipPath)).rejects.toThrow(
+        'No .sps file found in .sub.zip archive'
+      );
+
+      // Cleanup
+      fs.unlinkSync(invalidSubZipPath);
+    });
+
+    it('should throw error for .sub.zip file that does not exist', async () => {
+      const processor = new SnapProcessor();
+      await expect(processor.loadIntoTree('non-existent.sub.zip')).rejects.toThrow();
+    });
   });
 
   describe('Audio Options', () => {