From d5f454241f6dccc1046789b9b1d391139b6ee267 Mon Sep 17 00:00:00 2001 From: Noah Manneschmidt Date: Wed, 12 Aug 2026 21:44:01 -0700 Subject: [PATCH 1/3] first pass at support for parsing zips --- CHANGELOG.md | 5 + README.md | 40 ++- jest.config.mjs | 2 + package.json | 4 +- src/__tests__/browserZip.test.ts | 459 +++++++++++++++++++++++++++++++ src/__tests__/makeZip.ts | 211 ++++++++++++++ src/browser/index.ts | 360 ++++++++++++++++-------- src/browser/parseSong.ts | 266 +++--------------- src/browser/shared.ts | 56 ---- src/browser/vfs.ts | 397 ++++++++++++++++++++++++++ src/browser/zip.ts | 274 ++++++++++++++++++ tsconfig.json | 2 +- 12 files changed, 1673 insertions(+), 403 deletions(-) create mode 100644 src/__tests__/browserZip.test.ts create mode 100644 src/__tests__/makeZip.ts create mode 100644 src/browser/vfs.ts create mode 100644 src/browser/zip.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cc98ae..9ee24b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v0.10.0 + +- Added support for in-browser parsing of packs directly from a zip file, which is how packs are usually distributed. `parsePack` now accepts a dropped or selected zip in addition to a folder, and the new `parseZipPack` export takes a `File`/`Blob` directly. Archives are read lazily, so only the chart files and images are decompressed and the whole pack never has to be held in memory. Song folders may sit at the root of the archive or inside a pack folder. +- Fixed browser directory listings being truncated for large folders. + ## v0.9.0 - Exposed the subtitle tag in parsed results for better noCmod support (thanks Vincent!) diff --git a/README.md b/README.md index 53a3223..c2d175a 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,12 @@ [![npm](https://img.shields.io/npm/v/simfile-parser)](https://www.npmjs.com/package/simfile-parser) [![npm bundle size](https://img.shields.io/bundlephobia/min/simfile-parser)](https://bundlephobia.com/package/simfile-parser) -Original parsing code from [city41/stepcharts](https://github.com/city41/stepcharts). Props to Matt for building a really sweet site. +Parse stepmania simfiles in javascript with zero dependencies. Works both in node (server-side or CLI) and in browser. Reads individual songs, whole packs, groups of packs, or even a pack still inside a zip file. -Works both in node (server-side or CLI) and in browser. Bun and Deno support is untested, but an interesting future to explore! +Original parsing code from [city41/stepcharts](https://github.com/city41/stepcharts). Props to Matt for building a really sweet site. ## Usage -Install with `npm install --save simfile-parser` or `yarn add simfile-parser` - ```ts // in node.js >= 16.9.0 @@ -39,7 +37,8 @@ calculateStats(aGreatSong.charts["single-challenge"]); ### Browser support -Support dragging packs directly into a web app by parsing in-browser! +Support dragging packs directly into a web app by parsing in-browser! A pack +can be either a folder of song folders or a **zip file** containing one. ```ts // requires typescript 5.0 in "Bundler" module resolution mode for typings @@ -62,6 +61,7 @@ document.body.addEventListener("drop", async function (e) { } try { + // works for a dropped folder or a dropped .zip const pack = await parsePack(evt.dataTransfer.items[0]); console.log(`parsed pack "${pack.name}" with ${pack.songCount} songs`); } catch (e) { @@ -69,3 +69,33 @@ document.body.addEventListener("drop", async function (e) { } }); ``` + +#### Zipped packs + +`parsePack` detects zip files by content, so a pack dropped or selected as an +archive needs no unzipping first. You can also hand one straight to +`parseZipPack`, for example from a file input or a `fetch`: + +```ts +import { parseZipPack } from "simfile-parser/browser"; + +const response = await fetch("/packs/Club Fantastic Season 1.zip"); +const pack = await parseZipPack(await response.blob(), "Club Fantastic"); +``` + +Archives are read lazily: only the archive index, each song's chart file, and +its images are ever decompressed, so the audio and video that make up the bulk +of a pack are skipped entirely and the whole archive never has to be held in +memory. + +Only one pack per archive is supported. An archive holding several packs — a +whole `Songs` directory, say — throws rather than quietly parsing nothing: + +``` +expected an archive holding a single pack, but found 2: 'DDRMAX2', 'SuperNOVA2' +``` + +Reading zips uses [`DecompressionStream`][ds], which needs Chrome 103+, +Firefox 113+, or Safari 16.4+. Encrypted archives are not supported. + +[ds]: https://developer.mozilla.org/en-US/docs/Web/API/DecompressionStream diff --git a/jest.config.mjs b/jest.config.mjs index 027b9be..53b07fc 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -5,6 +5,8 @@ const tsJestCfg = createDefaultEsmPreset(); export default { ...tsJestCfg, testEnvironment: "node", + // otherwise shared test helpers get picked up as (empty) suites + testMatch: ["**/*.test.ts"], extensionsToTreatAsEsm: [".ts"], moduleNameMapper: { "^(\\.{1,2}/.*)\\.js$": "$1", diff --git a/package.json b/package.json index 043841c..3f52c26 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "simfile-parser", - "version": "0.9.0", + "version": "0.10.0-beta.0", "description": "Read stepmania charts with javascript!", "type": "module", "main": "./dist/main.js", @@ -11,7 +11,7 @@ "./browser": "./dist/browser/index.js" }, "bin": "./dist/cli.js", - "sideEffects": "false", + "sideEffects": false, "scripts": { "test": "NODE_OPTIONS=--experimental-vm-modules jest", "format": "prettier --write src/**/*.ts", diff --git a/src/__tests__/browserZip.test.ts b/src/__tests__/browserZip.test.ts new file mode 100644 index 0000000..6402a0d --- /dev/null +++ b/src/__tests__/browserZip.test.ts @@ -0,0 +1,459 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { parsePack as parsePackFromDisk } from "../main"; +import { parsePack, parseZipPack } from "../browser/index"; +import { openZip, isZip, DirLike, isDir } from "../browser/vfs"; +import { readCentralDirectory, readEntry } from "../browser/zip"; +import { makeZip, ZipFixtureFile, ZipFixtureOptions } from "./makeZip"; +import { setErrorTolerance } from "../util"; + +setErrorTolerance("bail"); + +const packsRoot = path.resolve(import.meta.dirname, "../../packs"); +const fixturePack = "Bhop Ball"; + +/** + * Reads a real pack off disk so it can be zipped up for the end to end tests. + * Audio is skipped to keep the fixtures small; it is never parsed anyway. + * @param packName name of a pack in the packs directory + * @param prefix path to nest the pack's contents under inside the archive + * @returns one entry per file in the pack + */ +function readPackFiles(packName: string, prefix = ""): ZipFixtureFile[] { + const root = path.join(packsRoot, packName); + const files: ZipFixtureFile[] = []; + const walk = (dir: string) => { + for (const child of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, child.name); + if (child.isDirectory()) { + walk(full); + } else if (!/\.(ogg|mp3|wav|avi|mpg)$/i.test(child.name)) { + files.push({ + name: prefix + path.relative(root, full).split(path.sep).join("/"), + data: new Uint8Array(fs.readFileSync(full)), + }); + } + } + }; + walk(root); + return files; +} + +/** + * @param packName name of a pack in the packs directory + * @param prefix path to nest the pack's contents under inside the archive + * @param options how to encode the archive + * @returns the pack as a zip file + */ +async function zipPack( + packName: string, + prefix = "", + options: ZipFixtureOptions = {}, +) { + const blob = await makeZip(readPackFiles(packName, prefix), options); + return new File([blob], `${packName}.zip`); +} + +/** + * Asserts something was found, so the tests can go on to use it without + * reaching for non-null assertions. + * @param value the possibly missing value + * @param what a description of what was being looked for + * @returns the value + */ +function found(value: T | null | undefined, what: string): T { + if (!value) { + throw new Error(`expected to find ${what}`); + } + return value; +} + +/** + * @param dir a virtual directory + * @returns the names of its children, sorted + */ +async function childNames(dir: DirLike) { + const names: string[] = []; + for await (const entry of dir.entries()) { + names.push(isDir(entry) ? `${entry.name}/` : entry.name); + } + return names.sort(); +} + +describe("zip format reader", () => { + const files: ZipFixtureFile[] = [ + { name: "song/steps.sm", data: "#TITLE:Test;" }, + { name: "song/banner.png", data: new Uint8Array([1, 2, 3, 4, 5]) }, + { name: "readme.txt", data: "x".repeat(5000) }, + ]; + + /** + * @param options how to encode the archive + * @returns the archive's entries mapped to their decoded contents + */ + async function roundTrip(options: ZipFixtureOptions) { + const blob = await makeZip(files, options); + const entries = await readCentralDirectory(blob); + const contents: Record = {}; + for (const entry of entries.filter((e) => !e.isDirectory)) { + const bytes = new Uint8Array( + await (await readEntry(blob, entry)).arrayBuffer(), + ); + contents[entry.name] = Array.from(bytes).join(","); + } + return contents; + } + + /** + * @returns the fixture files keyed the same way roundTrip returns them + */ + function expected() { + const contents: Record = {}; + for (const file of files) { + const bytes = + typeof file.data === "string" + ? new TextEncoder().encode(file.data) + : file.data; + contents[file.name] = Array.from(bytes).join(","); + } + return contents; + } + + test("reads deflated entries", async () => { + expect(await roundTrip({})).toEqual(expected()); + }); + + test("reads stored entries", async () => { + expect(await roundTrip({ stored: true })).toEqual(expected()); + }); + + test("reads zip64 archives", async () => { + expect(await roundTrip({ zip64: true })).toEqual(expected()); + }); + + test("reads archives with a trailing comment", async () => { + expect(await roundTrip({ comment: "made with some archiver" })).toEqual( + expected(), + ); + }); + + test("reads explicit directory entries", async () => { + const blob = await makeZip(files, { includeDirEntries: true }); + const entries = await readCentralDirectory(blob); + expect(entries.filter((e) => e.isDirectory).map((e) => e.name)).toEqual([ + "song/", + ]); + }); + + test("decodes utf-8 filenames", async () => { + const blob = await makeZip([{ name: "曲/ステップ.sm", data: "#TITLE:a;" }]); + const entries = await readCentralDirectory(blob); + expect(entries[0].name).toBe("曲/ステップ.sm"); + }); + + test("decodes cp437 filenames", async () => { + const blob = await makeZip([{ name: "Café/naïve.sm", data: "#TITLE:a;" }], { + cp437: true, + }); + const entries = await readCentralDirectory(blob); + expect(entries[0].name).toBe("Café/naïve.sm"); + }); + + test("rejects data that is not a zip", async () => { + const notAZip = new Blob(["this is definitely not a zip file"]); + expect(await isZip(notAZip)).toBe(false); + await expect(readCentralDirectory(notAZip)).rejects.toThrow( + /no end of central directory/, + ); + }); + + test("recognizes a real zip by its magic number", async () => { + expect(await isZip(await makeZip(files))).toBe(true); + }); +}); + +describe("openZip", () => { + /** + * @returns a small archive covering the shapes we care about + */ + function sampleArchive() { + return makeZip([ + { name: "Pack/Song One/steps.sm", data: "#TITLE:One;" }, + { name: "Pack/Song One/BANNER.png", data: "banner bytes" }, + { name: "Pack/Song Two/steps.sm", data: "#TITLE:Two;" }, + { name: "__MACOSX/Pack/._steps.sm", data: "junk" }, + { name: "Pack/.DS_Store", data: "junk" }, + ]); + } + + test("rebuilds the folder tree, including implied folders", async () => { + const root = await openZip(await sampleArchive(), "archive"); + expect(await childNames(root)).toEqual(["Pack/"]); + + const [pack] = [...(await entriesOf(root))]; + expect(await childNames(pack as DirLike)).toEqual([ + "Song One/", + "Song Two/", + ]); + }); + + test("drops archiver junk", async () => { + const root = await openZip(await sampleArchive(), "archive"); + expect(await childNames(root)).not.toContain("__MACOSX/"); + const pack = (await entriesOf(root))[0] as DirLike; + expect(await childNames(pack)).not.toContain(".DS_Store"); + }); + + test("resolves paths relative to a directory", async () => { + const root = await openZip(await sampleArchive(), "archive"); + const pack = (await entriesOf(root))[0] as DirLike; + const songOne = (await entriesOf(pack))[0] as DirLike; + + expect(await songOne.getFile("steps.sm")).toBeTruthy(); + expect(await songOne.getFile("nope.sm")).toBeNull(); + // walking back up out of the song folder + expect(await songOne.getFile("../Song Two/steps.sm")).toBeTruthy(); + // nested from the pack root + expect(await pack.getFile("Song Two/steps.sm")).toBeTruthy(); + // tags authored on windows sometimes use backslashes + expect(await pack.getFile("Song Two\\steps.sm")).toBeTruthy(); + }); + + test("resolves filenames case insensitively", async () => { + const root = await openZip(await sampleArchive(), "archive"); + const pack = (await entriesOf(root))[0] as DirLike; + const songOne = (await entriesOf(pack))[0] as DirLike; + // the simfile might tag this as banner.png while the archive has BANNER.png + const banner = found(await songOne.getFile("banner.png"), "banner.png"); + expect(await (await banner.file()).text()).toBe("banner bytes"); + }); + + test("only reads a given entry once", async () => { + const root = await openZip(await sampleArchive(), "archive"); + // an image often gets picked for more than one role in the same song + const first = found(await root.getFile("Pack/Song One/BANNER.png"), "once"); + const second = found( + await root.getFile("Pack/Song One/BANNER.png"), + "again", + ); + expect(first).not.toBe(second); + expect(await first.file()).toBe(await second.file()); + }); + + test("reads file contents lazily", async () => { + const root = await openZip(await sampleArchive(), "archive"); + const file = found( + await root.getFile("Pack/Song One/steps.sm"), + "steps.sm", + ); + expect(await (await file.file()).text()).toBe("#TITLE:One;"); + }); +}); + +/** + * @param dir a virtual directory + * @returns its children as an array + */ +async function entriesOf(dir: DirLike) { + const all = []; + for await (const entry of dir.entries()) { + all.push(entry); + } + return all; +} + +describe("parseZipPack", () => { + /** + * The node parser reads the same pack straight off disk, so it makes a good + * reference for what the zip parser ought to produce. + * @returns comparable fields for each song, sorted by title + */ + const fromDisk = () => + parsePackFromDisk(path.join(packsRoot, fixturePack)) + .simfiles.map((s) => ({ + title: s.title.titleName, + artist: s.artist, + minBpm: s.minBpm, + maxBpm: s.maxBpm, + displayBpm: s.displayBpm, + stopCount: s.stopCount, + charts: Object.keys(s.charts).sort(), + })) + .sort((a, b) => a.title.localeCompare(b.title)); + + /** + * @param pack a parsed pack + * @returns comparable fields for each song, sorted by title + */ + const comparable = (pack: Awaited>) => + pack.simfiles + .map((s) => ({ + title: s.title.titleName, + artist: s.artist, + minBpm: s.minBpm, + maxBpm: s.maxBpm, + displayBpm: s.displayBpm, + stopCount: s.stopCount, + charts: Object.keys(s.charts).sort(), + })) + .sort((a, b) => a.title.localeCompare(b.title)); + + test("matches the on-disk parser, for a pack wrapped in a folder", async () => { + const zip = await zipPack(fixturePack, `${fixturePack}/`); + const pack = await parseZipPack(zip); + expect(pack.songCount).toBe(2); + expect(pack.name).toBe(fixturePack); + expect(comparable(pack)).toEqual(fromDisk()); + }); + + test("matches the on-disk parser, for songs at the archive root", async () => { + const zip = await zipPack(fixturePack); + const pack = await parseZipPack(zip); + expect(pack.songCount).toBe(2); + // falls back to the archive's own filename for the pack name + expect(pack.name).toBe(fixturePack); + expect(comparable(pack)).toEqual(fromDisk()); + }); + + test("ignores mac metadata sitting alongside the pack folder", async () => { + // zips made on macos carry a __MACOSX folder next to the real one, which + // would otherwise leave the pack folder looking like one of two candidates + const blob = await makeZip([ + ...readPackFiles(fixturePack, `${fixturePack}/`), + { name: "__MACOSX/._" + fixturePack, data: "junk" }, + { name: `__MACOSX/${fixturePack}/._steps.sm`, data: "junk" }, + ]); + const pack = await parseZipPack(new File([blob], `${fixturePack}.zip`)); + expect(pack.name).toBe(fixturePack); + expect(comparable(pack)).toEqual(fromDisk()); + }); + + describe("a pack holding only one song", () => { + /** + * @param prefix path to nest the song under inside the archive + * @returns an archive containing a single song folder + */ + async function singleSongZip(prefix: string) { + const song = "[T10] Central Utopia"; + const files = readPackFiles(fixturePack, prefix).filter((f) => + f.name.includes(song), + ); + return new File([await makeZip(files)], "Solo Pack.zip"); + } + + // one song folder is the ambiguous case: a lone subfolder is normally a + // wrapper to descend through, but here it is the song itself + test("finds it at the archive root", async () => { + const pack = await parseZipPack(await singleSongZip("")); + expect(pack.songCount).toBe(1); + expect(pack.name).toBe("Solo Pack"); + }); + + test("finds it inside a pack folder", async () => { + const pack = await parseZipPack(await singleSongZip("Solo Pack/")); + expect(pack.songCount).toBe(1); + expect(pack.name).toBe("Solo Pack"); + }); + }); + + test("descends through several wrapper folders", async () => { + const zip = await zipPack(fixturePack, `downloads/new/${fixturePack}/`); + const pack = await parseZipPack(zip); + expect(pack.songCount).toBe(2); + expect(pack.name).toBe(fixturePack); + }); + + test("parses stored and zip64 archives the same way", async () => { + const reference = comparable( + await parseZipPack(await zipPack(fixturePack, `${fixturePack}/`)), + ); + for (const options of [{ stored: true }, { zip64: true }]) { + const zip = await zipPack(fixturePack, `${fixturePack}/`, options); + expect(comparable(await parseZipPack(zip))).toEqual(reference); + } + }); + + test("extracts images as files", async () => { + const zip = await zipPack(fixturePack, `${fixturePack}/`); + const pack = await parseZipPack(zip); + const song = found( + pack.simfiles.find((s) => s.title.titleDir.includes("Central Utopia")), + "Central Utopia", + ); + const bg = found(song.title.bg, "a background image"); + const banner = found(song.title.banner, "a banner image"); + expect(bg).toBeInstanceOf(File); + expect(banner).toBeInstanceOf(File); + // the real image bytes came through, not an empty placeholder + expect(bg.size).toBeGreaterThan(0); + expect(banner.size).toBeGreaterThan(0); + }); + + test("still finds the pack when junk folders sit beside it", async () => { + const blob = await makeZip([ + ...readPackFiles(fixturePack, `${fixturePack}/`), + { name: "_screenshots/shot.png", data: "not a song" }, + ]); + const pack = await parseZipPack(new File([blob], "download.zip")); + expect(pack.name).toBe(fixturePack); + expect(comparable(pack)).toEqual(fromDisk()); + }); + + describe("archives that aren't a single pack", () => { + /** + * @param packNames names of the packs to put in the archive + * @returns an archive containing a song folder under each named pack + */ + async function multiPackZip(packNames: string[]) { + const files = packNames.flatMap((packName) => [ + { name: `Songs/${packName}/A Song/steps.sm`, data: "#TITLE:A;" }, + ]); + return new File([await makeZip(files)], "Songs.zip"); + } + + test("refuses an archive holding more than one pack", async () => { + const zip = await multiPackZip(["Bhop Ball", "Club Fantastic"]); + await expect(parseZipPack(zip)).rejects.toThrow( + "expected an archive holding a single pack, but found 2: " + + "'Bhop Ball', 'Club Fantastic'", + ); + }); + + test("summarizes the rest when there are lots of packs", async () => { + const names = ["A", "B", "C", "D", "E", "F", "G"]; + await expect(parseZipPack(await multiPackZip(names))).rejects.toThrow( + "found 7: 'A', 'B', 'C', 'D', 'E', and 2 more", + ); + }); + + test("refuses an archive with no songs in it", async () => { + const blob = await makeZip([ + { name: "notes/readme.txt", data: "no charts here" }, + ]); + await expect(parseZipPack(new File([blob], "notes.zip"))).rejects.toThrow( + /found no songs in this archive/, + ); + }); + }); + + test("accepts an explicit pack name", async () => { + const zip = await zipPack(fixturePack, `${fixturePack}/`); + expect((await parseZipPack(zip, "Custom Name")).name).toBe("Custom Name"); + }); +}); + +describe("parsePack", () => { + test("accepts a zip file directly", async () => { + const zip = await zipPack(fixturePack, `${fixturePack}/`); + const pack = await parsePack(zip); + expect(pack.songCount).toBe(2); + expect(pack.name).toBe(fixturePack); + }); + + test("rejects a file that is not a zip or a folder", async () => { + const notAPack = new File(["#TITLE:lonely;"], "steps.sm"); + await expect(parsePack(notAPack)).rejects.toThrow( + /expected a folder or zip file/, + ); + }); +}); diff --git a/src/__tests__/makeZip.ts b/src/__tests__/makeZip.ts new file mode 100644 index 0000000..a14b378 --- /dev/null +++ b/src/__tests__/makeZip.ts @@ -0,0 +1,211 @@ +/** + * A minimal zip writer, used only to build fixtures for the zip reader tests. + * It deliberately supports the awkward variations real archives show up with: + * stored vs deflated entries, zip64, and cp437 vs utf-8 filenames. + */ + +const crcTable = (() => { + const table = new Uint32Array(256); + for (let i = 0; i < 256; i++) { + let c = i; + for (let bit = 0; bit < 8; bit++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[i] = c >>> 0; + } + return table; +})(); + +/** + * @param bytes data to checksum + * @returns the crc32 of the data + */ +function crc32(bytes: Uint8Array): number { + let crc = 0xffffffff; + for (const byte of bytes) { + crc = crcTable[(crc ^ byte) & 0xff] ^ (crc >>> 8); + } + return (crc ^ 0xffffffff) >>> 0; +} + +/** + * @param bytes data to compress + * @returns the raw deflate stream of the data + */ +async function deflate(bytes: Uint8Array): Promise { + const stream = new Blob([bytes as BlobPart]) + .stream() + .pipeThrough(new CompressionStream("deflate-raw")); + return new Uint8Array(await new Response(stream).arrayBuffer()); +} + +/** + * Encodes a filename as cp437, which for our fixtures only needs to cover the + * ascii range plus a couple of accented characters. + * @param name the filename + * @returns the encoded bytes + */ +function encodeCp437(name: string): Uint8Array { + const high = + "ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜ¢£¥₧ƒáíóúñѪº¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ "; + return Uint8Array.from( + [...name].map((char) => { + const code = char.charCodeAt(0); + if (code < 0x80) return code; + const index = high.indexOf(char); + if (index === -1) throw new Error(`cannot encode '${char}' as cp437`); + return index + 0x80; + }), + ); +} + +export interface ZipFixtureFile { + name: string; + data: string | Uint8Array; +} + +export interface ZipFixtureOptions { + /** store entries uncompressed (method 0) instead of deflating them */ + stored?: boolean; + /** write the archive as zip64, with the real sizes in extra fields */ + zip64?: boolean; + /** encode filenames as cp437 rather than flagging them as utf-8 */ + cp437?: boolean; + /** emit explicit entries for directories, which many archivers do */ + includeDirEntries?: boolean; + /** trailing archive comment, which the reader has to scan back past */ + comment?: string; +} + +const ZIP64_MARKER = 0xffffffff; + +/** + * Builds a zip archive in memory + * @param files the files to include + * @param options how to encode the archive + * @returns the archive as a blob + */ +export async function makeZip( + files: ZipFixtureFile[], + options: ZipFixtureOptions = {}, +): Promise { + const { stored, zip64, cp437, includeDirEntries, comment } = options; + const commentBytes = new TextEncoder().encode(comment ?? ""); + + const allNames = new Set(); + if (includeDirEntries) { + for (const file of files) { + const segments = file.name.split("/"); + segments.pop(); + for (let i = 1; i <= segments.length; i++) { + allNames.add(segments.slice(0, i).join("/") + "/"); + } + } + } + + const entries: { name: string; data: Uint8Array }[] = [ + ...[...allNames].map((name) => ({ name, data: new Uint8Array(0) })), + ...files.map((file) => ({ + name: file.name, + data: + typeof file.data === "string" + ? new TextEncoder().encode(file.data) + : file.data, + })), + ]; + + const local: Uint8Array[] = []; + const central: Uint8Array[] = []; + let offset = 0; + + for (const entry of entries) { + const isDirectory = entry.name.endsWith("/"); + const nameBytes = cp437 + ? encodeCp437(entry.name) + : new TextEncoder().encode(entry.name); + const method = stored || isDirectory ? 0 : 8; + const body = method === 0 ? entry.data : await deflate(entry.data); + const crc = crc32(entry.data); + const flags = cp437 ? 0 : 0x800; + + const header = new Uint8Array(30 + nameBytes.length); + const headerView = new DataView(header.buffer); + headerView.setUint32(0, 0x04034b50, true); + headerView.setUint16(4, 20, true); + headerView.setUint16(6, flags, true); + headerView.setUint16(8, method, true); + headerView.setUint32(14, crc, true); + headerView.setUint32(18, body.length, true); + headerView.setUint32(22, entry.data.length, true); + headerView.setUint16(26, nameBytes.length, true); + header.set(nameBytes, 30); + local.push(header, body); + + // zip64 entries hide their real sizes and offset in an extra field + const extra = new Uint8Array(zip64 ? 28 : 0); + if (zip64) { + const extraView = new DataView(extra.buffer); + extraView.setUint16(0, 0x0001, true); + extraView.setUint16(2, 24, true); + extraView.setBigUint64(4, BigInt(entry.data.length), true); + extraView.setBigUint64(12, BigInt(body.length), true); + extraView.setBigUint64(20, BigInt(offset), true); + } + + const record = new Uint8Array(46 + nameBytes.length + extra.length); + const recordView = new DataView(record.buffer); + recordView.setUint32(0, 0x02014b50, true); + recordView.setUint16(4, 20, true); + recordView.setUint16(6, 20, true); + recordView.setUint16(8, flags, true); + recordView.setUint16(10, method, true); + recordView.setUint32(16, crc, true); + recordView.setUint32(20, zip64 ? ZIP64_MARKER : body.length, true); + recordView.setUint32(24, zip64 ? ZIP64_MARKER : entry.data.length, true); + recordView.setUint16(28, nameBytes.length, true); + recordView.setUint16(30, extra.length, true); + recordView.setUint32(42, zip64 ? ZIP64_MARKER : offset, true); + record.set(nameBytes, 46); + record.set(extra, 46 + nameBytes.length); + central.push(record); + + offset += header.length + body.length; + } + + const centralSize = central.reduce((sum, r) => sum + r.length, 0); + const centralOffset = offset; + const tail: Uint8Array[] = []; + + if (zip64) { + const record = new Uint8Array(56); + const recordView = new DataView(record.buffer); + recordView.setUint32(0, 0x06064b50, true); + recordView.setBigUint64(4, 44n, true); + recordView.setBigUint64(24, BigInt(entries.length), true); + recordView.setBigUint64(32, BigInt(entries.length), true); + recordView.setBigUint64(40, BigInt(centralSize), true); + recordView.setBigUint64(48, BigInt(centralOffset), true); + + const locator = new Uint8Array(20); + const locatorView = new DataView(locator.buffer); + locatorView.setUint32(0, 0x07064b50, true); + locatorView.setBigUint64(8, BigInt(centralOffset + centralSize), true); + locatorView.setUint32(16, 1, true); + + tail.push(record, locator); + } + + const eocd = new Uint8Array(22); + const eocdView = new DataView(eocd.buffer); + eocdView.setUint32(0, 0x06054b50, true); + eocdView.setUint16(8, zip64 ? 0xffff : entries.length, true); + eocdView.setUint16(10, zip64 ? 0xffff : entries.length, true); + eocdView.setUint32(12, zip64 ? ZIP64_MARKER : centralSize, true); + eocdView.setUint32(16, zip64 ? ZIP64_MARKER : centralOffset, true); + eocdView.setUint16(20, commentBytes.length, true); + tail.push(eocd, commentBytes); + + return new Blob( + [...local, ...central, ...tail].map((part) => part as BlobPart), + ); +} diff --git a/src/browser/index.ts b/src/browser/index.ts index d285e9a..4d010f8 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -1,44 +1,8 @@ +import { supportedExtensions } from "../parsers/index.js"; import { Pack } from "../types.js"; import { reportError } from "../util.js"; import { BrowserSimfile, parseSong } from "./parseSong.js"; -import { - AnyFileOrEntry, - isDirectoryEntry, - isDirectoryHandle, -} from "./shared.js"; - -/** - * @param dir directory handle - * @yields {FileSystemDirectoryHandle | FileSystemDirectoryEntry} for each subdir of the given dir - * @returns nothing - */ -async function* getDirectories( - dir: FileSystemDirectoryHandle | FileSystemDirectoryEntry, -) { - if ("createReader" in dir) { - const dirs = await getDirectoriesFromEntry(dir); - yield* dirs; - } else { - for await (const child of dir.values()) { - if (child.kind === "directory") { - yield child; - } - } - } -} - -/** - * @param dir file system entry - * @returns only subdirectories of the given directory - */ -function getDirectoriesFromEntry(dir: FileSystemDirectoryEntry) { - const dirReader = dir.createReader(); - return new Promise((resolve, reject) => { - dirReader.readEntries((results) => { - resolve(results.filter(isDirectoryEntry)); - }, reject); - }); -} +import { AnyEntry, DirLike, fromDom, isDir, isZip, openZip } from "./vfs.js"; declare global { interface DataTransferItem { @@ -49,62 +13,188 @@ declare global { export type PackWithSongs = Pack & { simfiles: BrowserSimfile[] }; +export type { BrowserSimfile, BrowserTitle } from "./parseSong.js"; + /** - * Parse a pack drag/dropped by a user in a browser - * @param item a DataTransferItem from a drop event - * @returns parsed pack + * Pulls a usable file/folder reference out of whatever the browser handed us. + * @param item a dropped item, a file input, or a file + * @returns the item as a virtual filesystem entry */ -export async function parsePack(item: DataTransferItem | HTMLInputElement) { - let dir: FileSystemDirectoryEntry | FileSystemDirectoryHandle; +async function resolveItem( + item: DataTransferItem | HTMLInputElement | File, +): Promise { + if (item instanceof File) { + return fromDom(item); + } if (item instanceof HTMLInputElement) { - if ("webkitEntries" in item) { - const entries = item.webkitEntries; - if (entries.length !== 1) { + if ("webkitEntries" in item && item.webkitEntries.length) { + if (item.webkitEntries.length > 1) { throw new Error("expected exactly one selected file"); } - const entry = entries[0]; - if (!isDirectoryEntry(entry)) { - throw new Error("expected folder to be dropped, but got file"); + return fromDom(item.webkitEntries[0]); + } + if (item.files?.length) { + if (item.files.length > 1) { + throw new Error("expected exactly one selected file"); } - dir = entry; - } else { - throw new Error("entries property not available on provided input"); + return fromDom(item.files[0]); } - } else { - if (item.kind !== "file") { - throw new Error("expected file to be dropped, but it was not a file"); + throw new Error("no files available on provided input"); + } + if (item.kind !== "file") { + throw new Error("expected file to be dropped, but it was not a file"); + } + if (item.getAsFileSystemHandle) { + const handle = await item.getAsFileSystemHandle(); + if (!handle) { + throw new Error("could not get file handle from drop item"); } - if (item.getAsFileSystemHandle) { - const dirHandle = await item.getAsFileSystemHandle(); - if (!dirHandle) { - throw new Error("could not get file handle from drop item"); - } - if (!isDirectoryHandle(dirHandle)) { - throw new Error("expected folder to be dropped, but got file"); - } - dir = dirHandle; - } else if ("webkitGetAsEntry" in item) { - const entry = item.webkitGetAsEntry(); - if (!entry) { - throw new Error("could not get a file entry from drop item"); - } - if (!isDirectoryEntry(entry)) { - throw new Error("expected folder to be dropped, but got file"); - } - dir = entry; - } else { - throw new Error("no supported file drop mechanism supported"); + return fromDom(handle); + } + if ("webkitGetAsEntry" in item) { + const entry = item.webkitGetAsEntry(); + if (!entry) { + throw new Error("could not get a file entry from drop item"); + } + return fromDom(entry); + } + throw new Error("no supported file drop mechanism supported"); +} + +/** + * @param dir a directory to inspect + * @returns true if the directory directly contains a simfile + */ +async function containsSimfile(dir: DirLike): Promise { + for await (const entry of dir.entries()) { + if ( + !isDir(entry) && + supportedExtensions.some((ext) => entry.name.endsWith(ext)) + ) { + return true; + } + } + return false; +} + +/** + * @param dir a directory to inspect + * @returns the directory's immediate subfolders + */ +async function subdirectories(dir: DirLike): Promise { + const subdirs: DirLike[] = []; + for await (const entry of dir.entries()) { + if (isDir(entry)) { + subdirs.push(entry); + } + } + return subdirs; +} + +/** + * @param dir a directory to inspect + * @returns true if any of the directory's subfolders is a song folder + */ +async function looksLikePack(dir: DirLike): Promise { + for (const subdir of await subdirectories(dir)) { + if (await containsSimfile(subdir)) { + return true; + } + } + return false; +} + +/** how many nested wrapper folders to look through before giving up */ +const maxPackDepth = 4; + +type PackSearch = + | { type: "found"; dir: DirLike } + | { type: "multiple"; packs: DirLike[] } + | { type: "none" }; + +/** + * Finds the folder that actually holds the song folders. Archives commonly + * wrap a pack in one or more extra folders, so descend through them until we + * reach a folder whose children look like songs. + * @param dir the root of the archive + * @param depth how many levels have been descended so far + * @returns the pack folder, or why one couldn't be settled on + */ +async function findPackRoot(dir: DirLike, depth = 0): Promise { + if (await looksLikePack(dir)) { + return { type: "found", dir }; + } + + // nothing here is a song, so look for the pack among the subfolders. Doing + // it by what they contain rather than by counting them means junk folders + // sitting next to the pack don't make it ambiguous. + const subdirs = await subdirectories(dir); + const packs: DirLike[] = []; + for (const subdir of subdirs) { + if (await looksLikePack(subdir)) { + packs.push(subdir); } } + if (packs.length === 1) { + return { type: "found", dir: packs[0] }; + } + if (packs.length > 1) { + return { type: "multiple", packs }; + } - const pack: Pack = { - name: dir.name.replace(/-/g, " "), - dir: dir.name, + if (subdirs.length === 1 && depth < maxPackDepth) { + return findPackRoot(subdirs[0], depth + 1); + } + return { type: "none" }; +} + +/** how many pack names to name individually before summarizing the rest */ +const maxNamesInError = 5; + +/** + * @param packs the packs found in an archive + * @returns an error explaining that only one pack can be parsed at a time + */ +function multiplePacksError(packs: DirLike[]): Error { + // sorted so the message doesn't depend on the order the archive happens to + // list its entries in + const names = packs.map((pack) => `'${pack.name}'`).sort(); + const listed = names.slice(0, maxNamesInError).join(", "); + const rest = names.length - maxNamesInError; + return new Error( + `expected an archive holding a single pack, but found ${names.length}: ` + + (rest > 0 ? `${listed}, and ${rest} more` : listed), + ); +} + +/** + * @param dirName the name of the folder a pack was found in + * @returns pack metadata derived from that folder name + */ +function packFromDirName(dirName: string): Pack { + return { + name: dirName.replace(/-/g, " "), + dir: dirName, songCount: 0, }; +} + +/** + * Parses every song folder inside a directory into a pack + * @param dir the pack's folder + * @param pack metadata for the pack being built, mutated with the song count + * @returns parsed pack + */ +async function parsePackDir(dir: DirLike, pack: Pack): Promise { + const songFolders: DirLike[] = []; + for await (const entry of dir.entries()) { + if (isDir(entry)) { + songFolders.push(entry); + } + } const simfiles: BrowserSimfile[] = []; - for await (const songFolder of getDirectories(dir)) { + for (const songFolder of songFolders) { try { const songData = await parseSong(songFolder); if (songData) { @@ -120,55 +210,89 @@ export async function parsePack(item: DataTransferItem | HTMLInputElement) { pack.songCount = simfiles.length; - return { + return { ...pack, simfiles, }; } +/** + * @param filename name of a zip file + * @returns the name with any `.zip` extension removed + */ +function stripZipExtension(filename: string) { + return filename.replace(/\.zip$/i, ""); +} + +/** + * Parse a pack directly from a zip archive, without unzipping it first. + * + * Only the archive's index and the files belonging to each song are read, so + * large packs don't have to be held in memory all at once. + * @param archive the zip file + * @param name optional pack name. Defaults to the name of the folder the songs + * were found in, falling back to the archive's own filename. + * @throws {Error} if the archive holds more than one pack, or no songs at all + * @returns parsed pack + */ +export async function parseZipPack( + archive: File | Blob, + name?: string, +): Promise { + const archiveName = + archive instanceof File ? stripZipExtension(archive.name) : ""; + const root = await openZip(archive, archiveName); + + const search = await findPackRoot(root); + if (search.type === "multiple") { + throw multiplePacksError(search.packs); + } + if (search.type === "none") { + throw new Error( + "found no songs in this archive; expected a pack containing one folder per song", + ); + } + const packDir = search.dir; + const dirName = packDir.name || archiveName; + return parsePackDir( + packDir, + // an explicitly provided name is used as given, rather than being run + // through the guesswork we apply to folder names + name ? { name, dir: dirName, songCount: 0 } : packFromDirName(dirName), + ); +} + +/** + * Parse a pack drag/dropped by a user in a browser. The pack may be either a + * folder of song folders or a zip archive containing one. + * @param item a DataTransferItem from a drop event, a file input, or a file + * @returns parsed pack + */ +export async function parsePack( + item: DataTransferItem | HTMLInputElement | File, +): Promise { + const entry = await resolveItem(item); + + if (!isDir(entry)) { + const file = await entry.file(); + if (!(await isZip(file))) { + throw new Error("expected a folder or zip file, but got another file"); + } + // let parseZipPack name the pack after the folder it finds the songs in, + // falling back to the archive's filename + return parseZipPack(file); + } + + return parsePackDir(entry, packFromDirName(entry.name)); +} + /** * For parsing a single song instead. Parses either a whole song folder, or just the metadata from a single simfile (ssc/sm/dwi) * @param item a data transfer item or HTML Input element a user has added a file selection to * @returns a simfile or null */ export async function parseSongFolderOrData( - item: DataTransferItem | HTMLInputElement, + item: DataTransferItem | HTMLInputElement | File, ): Promise { - let dirOrFile: FileSystemEntry | FileSystemHandle | File; - if (item instanceof HTMLInputElement) { - if ("webkitEntries" in item && item.webkitEntries.length > 0) { - const entries = item.webkitEntries; - if (entries.length > 1) { - throw new Error("expected exactly one selected file"); - } - dirOrFile = entries[0]; - } else if (item.files?.length) { - if (item.files.length > 1) { - throw new Error("expected exactly one selected file"); - } - dirOrFile = item.files[0]; - } else { - throw new Error("no files available on provided input"); - } - } else { - if (item.kind !== "file") { - throw new Error("expected file to be dropped, but it was not a file"); - } - if (item.getAsFileSystemHandle) { - const dirHandle = await item.getAsFileSystemHandle(); - if (!dirHandle) { - throw new Error("could not get file handle from drop item"); - } - dirOrFile = dirHandle; - } else if ("webkitGetAsEntry" in item) { - const entry = item.webkitGetAsEntry(); - if (!entry) { - throw new Error("could not get a file entry from drop item"); - } - dirOrFile = entry; - } else { - throw new Error("no supported file drop mechanism supported"); - } - } - return parseSong(dirOrFile as AnyFileOrEntry); + return parseSong(await resolveItem(item)); } diff --git a/src/browser/parseSong.ts b/src/browser/parseSong.ts index 070c4c5..f00f323 100644 --- a/src/browser/parseSong.ts +++ b/src/browser/parseSong.ts @@ -5,200 +5,61 @@ import { } from "../parsers/index.js"; import { ParsedImages, RawSimfile } from "../parsers/types.js"; import { Simfile, Title } from "../types.js"; -import { extname, isAnyDirectory, isFileEntry } from "./shared.js"; +import { extname } from "./shared.js"; +import { AnyEntry, DirLike, FileLike, isDir } from "./vfs.js"; /** - * @param files a list of candidate files to use for song info - * @returns the the most preferred candidate file + * Find the best simfile in a given directory + * @param songDir directory to search + * @returns the most preferred simfile found, or null */ -function getBestSongFileMatch< - T extends FileSystemFileEntry | FileSystemFileHandle, ->(files: T[]): T | null { - if (!files.length) { - return null; - } - files.sort((a, b) => { - return sortFileCandidatesByPriority(a.name, b.name); - }); - return files[0]; -} - -/** - * Find a simfile in a given directory - * @param songDir directory handle - * @returns file handle for the song file - */ -async function identifySongFile( - songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry, -) { - if ("createReader" in songDir) { - const candidates = await getSongFilesFromEntry(songDir); - return getBestSongFileMatch(candidates); - } - - const candidates: FileSystemFileHandle[] = []; - for await (const handle of songDir.values()) { - if (handle.kind === "file") { - if (supportedExtensions.some((ext) => handle.name.endsWith(ext))) { - candidates.push(handle); - } +async function identifySongFile(songDir: DirLike): Promise { + const candidates: FileLike[] = []; + for await (const entry of songDir.entries()) { + if ( + !isDir(entry) && + supportedExtensions.some((ext) => entry.name.endsWith(ext)) + ) { + candidates.push(entry); } } - return getBestSongFileMatch(candidates); -} - -/** - * @param songDir legacy file system entry - * @returns promise of the found file entry or null - */ -async function getSongFilesFromEntry(songDir: FileSystemDirectoryEntry) { - const dirReader = songDir.createReader(); - return new Promise((resolve, reject) => { - dirReader.readEntries((results) => { - const ret: FileSystemFileEntry[] = []; - for (const result of results) { - if (isFileEntry(result)) { - if (supportedExtensions.some((ext) => result.name.endsWith(ext))) { - ret.push(result); - } - } - } - resolve(ret); - }, reject); - }); + if (!candidates.length) { + return null; + } + candidates.sort((a, b) => sortFileCandidatesByPriority(a.name, b.name)); + return candidates[0]; } const imageExts = new Set([".png", ".jpg"]); /** * Get all image files in a given directory - * @param songDir directory - * @yields {FileSystemDirectoryHandle | FileSystemDirectoryEntry} file handles filtered to supported image extentions + * @param songDir directory to search + * @yields {FileLike} each file with a supported image extension */ -async function* getImages( - songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry, -) { - let files: - | AsyncIterable - | Iterable; - if ("values" in songDir) { - files = songDir.values(); - } else { - files = await new Promise((res, rej) => - songDir.createReader().readEntries(res, rej), - ); - } - for await (const file of files) { - const ext = extname(file.name); - if (!ext) { +async function* getImages(songDir: DirLike) { + for await (const entry of songDir.entries()) { + if (isDir(entry)) { continue; } - if ("kind" in file && file.kind === "directory") { - continue; - } - if ("isFile" in file && !isFileEntry(file)) { - continue; - } - if (imageExts.has(ext)) { - yield file as FileSystemFileEntry | FileSystemFileHandle; + const ext = extname(entry.name); + if (ext && imageExts.has(ext)) { + yield entry; } } } -/** - * gets file handle/entry from a directory by name - * @param dir directory handle or entry - * @param name name of file to get - * @returns promise of a handle or entry - */ -async function getByName(dir: DirRef, name: string) { - if (name.startsWith("../")) { - if ("getFileHandle" in dir) { - throw new Error("no way to resolve upward relative paths using this api"); - } else { - const parent = (await new Promise( - dir.getParent, - )) as FileSystemDirectoryEntry; - return getByName(parent, name.slice(3)); - } - } - if ("getFileHandle" in dir) { - return dir.getFileHandle(name); - } else { - return new Promise((resolve, reject) => - dir.getFile( - name, - {}, - (e) => { - if (isFileEntry(e)) { - resolve(e); - } else { - reject("file was not usable?"); - } - }, - reject, - ), - ); - } -} - -/** - * returns a file object from a handle/entry - * @param f the file handle or file entry - * @returns promise of File object - */ -async function getFileContents(f: FileSystemFileHandle | FileSystemFileEntry) { - if ("getFile" in f) { - return f.getFile(); - } else { - return new Promise((res, reject) => { - const a = true; - f.file(res, (reason) => { - debugger; - reject(reason); - }); - return a; - }); - } -} - -type DirRef = FileSystemDirectoryHandle | FileSystemDirectoryEntry; -type FileRef = FileSystemFileHandle | FileSystemFileEntry; - -/** - * Same as above, but catches and reports the error - * @param dir directory reference - * @param name file name or path - * @returns promise of directory or null - */ -async function guardedGetByName(dir: DirRef, name: string) { - try { - return await getByName(dir, name); - } catch { - return null; - } -} - /** * Make some best guesses about which images should be used for which fields - * @param songDir path to a song directory + * @param songDir the song's directory * @param tagged image metadata found in simfile * @returns final image metadata */ -async function guessImages( - songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry, - tagged: ParsedImages, -) { - let jacket: FileRef | null = tagged.jacket - ? await guardedGetByName(songDir, tagged.jacket) - : null; - let bg: FileRef | null = tagged.bg - ? await guardedGetByName(songDir, tagged.bg) - : null; - let banner: FileRef | null = tagged.banner - ? await guardedGetByName(songDir, tagged.banner) - : null; - const leftovers: FileRef[] = []; +async function guessImages(songDir: DirLike, tagged: ParsedImages) { + let jacket = tagged.jacket ? await songDir.getFile(tagged.jacket) : null; + let bg = tagged.bg ? await songDir.getFile(tagged.bg) : null; + let banner = tagged.banner ? await songDir.getFile(tagged.banner) : null; + const leftovers: FileLike[] = []; for await (const image of getImages(songDir)) { const imageName = image.name; const ext = extname(imageName) || ""; @@ -231,9 +92,9 @@ async function guessImages( jacket = leftovers.shift() || null; } return { - jacket: jacket ? await getFileContents(jacket) : null, - bg: bg ? await getFileContents(bg) : null, - banner: banner ? await getFileContents(banner) : null, + jacket: jacket ? await jacket.file() : null, + bg: bg ? await bg.file() : null, + banner: banner ? await banner.file() : null, }; } @@ -257,58 +118,21 @@ export type BrowserSimfile = Omit & { title: BrowserTitle; }; -/** - * given a directory or file handle/entry, get the actual File instance for the simfile - * @param entryOrHandle either a handle or entry type reference to a directory or file - * @returns File or null if not found - */ -async function getFileInstance( - entryOrHandle: - | FileSystemDirectoryHandle - | FileSystemDirectoryEntry - | FileSystemFileHandle - | FileSystemFileEntry, -): Promise { - let songFileHandleOrEntry: FileSystemFileHandle | FileSystemFileEntry; - if (isAnyDirectory(entryOrHandle)) { - const identified = await identifySongFile(entryOrHandle); - if (!identified) { - return null; - } - songFileHandleOrEntry = identified; - } else { - songFileHandleOrEntry = entryOrHandle; - } - - let ret: File; - if ("getFile" in songFileHandleOrEntry) { - ret = await songFileHandleOrEntry.getFile(); - } else { - ret = await new Promise((resolve, reject) => - songFileHandleOrEntry.file(resolve, reject), - ); - } - return ret; -} - /** * Parse a single simfile by folder or individual file. Automatically determines which parser to use depending on chart definition type. * @param songDirOrFile song folder or file reference (contains a chart definition file [dwi/sm/ssc], images, etc) * @returns a simfile object without mix info or null if no sm/ssc file was found */ export async function parseSong( - songDirOrFile: - | FileSystemDirectoryHandle - | FileSystemDirectoryEntry - | FileSystemFileHandle - | FileSystemFileEntry - | File, + songDirOrFile: AnyEntry, ): Promise { - const file = - songDirOrFile instanceof File - ? songDirOrFile - : await getFileInstance(songDirOrFile); - if (!file) return null; + const songDir = isDir(songDirOrFile) ? songDirOrFile : null; + const songFile = songDir + ? await identifySongFile(songDir) + : (songDirOrFile as FileLike); + if (!songFile) return null; + + const file = await songFile.file(); const extension = extname(file.name); if (!extension) return null; @@ -335,8 +159,8 @@ export async function parseSong( displayBpm = minBpm === maxBpm ? minBpm.toString() : `${minBpm}-${maxBpm}`; } - const finalImages = isAnyDirectory(songDirOrFile) - ? await guessImages(songDirOrFile, images) + const finalImages = songDir + ? await guessImages(songDir, images) : { banner: null, bg: null, jacket: null }; return { diff --git a/src/browser/shared.ts b/src/browser/shared.ts index 29726aa..aede96c 100644 --- a/src/browser/shared.ts +++ b/src/browser/shared.ts @@ -10,59 +10,3 @@ export function extname(name: string) { } return null; } - -/** - * narrows the type of a file system entry - * @param handle file system entry - * @param handle.isFile anything - * @returns true if entry is a file - */ -export function isFileEntry(handle: { - isFile: boolean; -}): handle is FileSystemFileEntry { - return handle.isFile; -} - -export type AnyFileOrEntry = - | FileSystemFileEntry - | FileSystemDirectoryEntry - | FileSystemFileHandle - | FileSystemDirectoryHandle; - -/** - * given any handle or entry, determine if it is a directory - * @param handleOrEntry a file system handle or entry - * @returns true if directory - */ -export function isAnyDirectory( - handleOrEntry: FileSystemHandle | FileSystemEntry | File, -): handleOrEntry is FileSystemDirectoryHandle | FileSystemDirectoryEntry { - if ("kind" in handleOrEntry) { - return handleOrEntry.kind === "directory"; - } else if ("isDirectory" in handleOrEntry) { - return handleOrEntry.isDirectory; - } - return false; -} - -/** - * narrows the type of a file system handle - * @param handle file system handle - * @returns true if handle is a dir - */ -export function isDirectoryHandle( - handle: FileSystemHandle, -): handle is FileSystemDirectoryHandle { - return handle.kind === "directory"; -} - -/** - * narrows the type of a file system handle - * @param handle file system handle - * @returns true if handle is a dir - */ -export function isDirectoryEntry( - handle: FileSystemEntry, -): handle is FileSystemDirectoryEntry { - return handle.isDirectory; -} diff --git a/src/browser/vfs.ts b/src/browser/vfs.ts new file mode 100644 index 0000000..adcdaee --- /dev/null +++ b/src/browser/vfs.ts @@ -0,0 +1,397 @@ +/** + * A tiny virtual filesystem the parsers can target, so they don't have to care + * whether a song came from a `FileSystemDirectoryHandle`, the older + * `FileSystemDirectoryEntry`, or a folder inside a zip archive. + */ + +import { readCentralDirectory, readEntry, ZipEntry } from "./zip.js"; + +export interface FileLike { + type: "file"; + name: string; + /** reads the file's contents */ + file(): Promise; +} + +export interface DirLike { + type: "directory"; + name: string; + /** iterates the directory's immediate children */ + entries(): AsyncIterable; + /** + * resolves a path relative to this directory, which may include `..` + * segments, to a file. Resolves to null if it can't be found. + */ + getFile(path: string): Promise; +} + +export type AnyEntry = FileLike | DirLike; + +/** + * @param entry any virtual filesystem entry + * @returns true if the entry is a directory + */ +export function isDir(entry: AnyEntry): entry is DirLike { + return entry.type === "directory"; +} + +/** + * Splits a simfile-relative path into segments, tolerating the backslashes + * that occasionally show up in tags authored on Windows. + * @param path a relative path + * @returns the meaningful path segments + */ +function splitPath(path: string): string[] { + return path.split(/[/\\]/).filter((segment) => segment && segment !== "."); +} + +// --- File System Access API (handles) --------------------------------------- + +/** + * @param handle a file handle + * @returns the handle as a virtual filesystem file + */ +function fileFromHandle(handle: FileSystemFileHandle): FileLike { + return { + type: "file", + name: handle.name, + file: () => handle.getFile(), + }; +} + +/** + * @param handle a directory handle + * @returns the handle as a virtual filesystem directory + */ +function dirFromHandle(handle: FileSystemDirectoryHandle): DirLike { + return { + type: "directory", + name: handle.name, + async *entries() { + for await (const child of handle.values()) { + yield fromHandle(child); + } + }, + async getFile(path) { + const segments = splitPath(path); + const filename = segments.pop(); + if (!filename) { + return null; + } + try { + let dir = handle; + for (const segment of segments) { + if (segment === "..") { + // this api gives no way to walk up out of the granted directory + return null; + } + dir = await dir.getDirectoryHandle(segment); + } + return fileFromHandle(await dir.getFileHandle(filename)); + } catch { + return null; + } + }, + }; +} + +/** + * @param handle any file system handle + * @returns the handle as a virtual filesystem entry + */ +export function fromHandle(handle: FileSystemHandle): AnyEntry { + return handle.kind === "directory" + ? dirFromHandle(handle as FileSystemDirectoryHandle) + : fileFromHandle(handle as FileSystemFileHandle); +} + +// --- legacy drag & drop entries --------------------------------------------- + +/** + * @param entry a file entry + * @returns the entry as a virtual filesystem file + */ +function fileFromEntry(entry: FileSystemFileEntry): FileLike { + return { + type: "file", + name: entry.name, + file: () => new Promise((resolve, reject) => entry.file(resolve, reject)), + }; +} + +/** + * `readEntries` only returns a limited number of children per call, so it has + * to be called until it comes back empty to see a whole directory. + * @param dir a directory entry + * @returns every child of the directory + */ +function readAllEntries(dir: FileSystemDirectoryEntry) { + const reader = dir.createReader(); + const all: FileSystemEntry[] = []; + return new Promise((resolve, reject) => { + const readBatch = () => + reader.readEntries((batch) => { + if (!batch.length) { + resolve(all); + return; + } + all.push(...batch); + readBatch(); + }, reject); + readBatch(); + }); +} + +/** + * @param entry a directory entry + * @returns the entry as a virtual filesystem directory + */ +function dirFromEntry(entry: FileSystemDirectoryEntry): DirLike { + return { + type: "directory", + name: entry.name, + async *entries() { + for (const child of await readAllEntries(entry)) { + yield fromEntry(child); + } + }, + async getFile(path) { + const segments = splitPath(path); + try { + let dir = entry; + while (segments[0] === "..") { + segments.shift(); + dir = await new Promise((resolve, reject) => + dir.getParent(resolve as never, reject), + ); + } + if (!segments.length) { + return null; + } + const found = await new Promise((resolve, reject) => + dir.getFile(segments.join("/"), {}, resolve, reject), + ); + return found.isFile + ? fileFromEntry(found as FileSystemFileEntry) + : null; + } catch { + return null; + } + }, + }; +} + +/** + * @param entry any file system entry + * @returns the entry as a virtual filesystem entry + */ +export function fromEntry(entry: FileSystemEntry): AnyEntry { + return entry.isDirectory + ? dirFromEntry(entry as FileSystemDirectoryEntry) + : fileFromEntry(entry as FileSystemFileEntry); +} + +// --- plain files ------------------------------------------------------------ + +/** + * @param file a file + * @returns the file as a virtual filesystem file + */ +export function fromFile(file: File): FileLike { + return { + type: "file", + name: file.name, + file: () => Promise.resolve(file), + }; +} + +/** + * @param source anything a browser might hand us for a dropped item + * @returns the item as a virtual filesystem entry + */ +export function fromDom( + source: FileSystemHandle | FileSystemEntry | File, +): AnyEntry { + if (source instanceof File) { + return fromFile(source); + } + return "kind" in source ? fromHandle(source) : fromEntry(source); +} + +// --- zip archives ----------------------------------------------------------- + +/** folders some archivers add alongside the real contents */ +const ignoredNames = new Set(["__MACOSX", ".DS_Store", "Thumbs.db"]); + +interface ZipNode { + name: string; + dirs: Map; + files: Map; + parent: ZipNode | null; +} + +/** + * @param name the node's own name + * @param parent the containing node, if any + * @returns an empty tree node + */ +function makeNode(name: string, parent: ZipNode | null): ZipNode { + return { name, dirs: new Map(), files: new Map(), parent }; +} + +/** + * Rebuilds the archive's folder hierarchy from its flat list of entries. + * Intermediate folders are created as needed, since archives are not required + * to include explicit entries for them. + * @param entries every entry in the archive + * @param rootName a name to give the root of the tree + * @returns the root node of the tree + */ +function buildTree(entries: ZipEntry[], rootName: string): ZipNode { + const root = makeNode(rootName, null); + for (const entry of entries) { + const segments = splitPath(entry.name); + if (!segments.length || segments.some((s) => ignoredNames.has(s))) { + continue; + } + const filename = entry.isDirectory ? null : segments.pop(); + let node = root; + for (const segment of segments) { + let child = node.dirs.get(segment); + if (!child) { + child = makeNode(segment, node); + node.dirs.set(segment, child); + } + node = child; + } + if (filename && !ignoredNames.has(filename)) { + node.files.set(filename, entry); + } + } + return root; +} + +/** + * Looks up a key in a map, falling back to a case insensitive match. Packs are + * routinely authored on case insensitive filesystems, so a simfile's tags may + * disagree with the archive on the casing of a filename. + * @param map the map to search + * @param key the key to look for + * @returns the matching value, or undefined + */ +function lenientGet(map: Map, key: string): T | undefined { + const exact = map.get(key); + if (exact !== undefined) { + return exact; + } + const lowered = key.toLowerCase(); + for (const [candidate, value] of map) { + if (candidate.toLowerCase() === lowered) { + return value; + } + } + return undefined; +} + +/** + * @param archive the source archive + * @param node the tree node to wrap + * @returns the node as a virtual filesystem directory + */ +function dirFromZipNode(archive: Blob, node: ZipNode): DirLike { + return { + type: "directory", + name: node.name, + async *entries() { + for (const child of node.dirs.values()) { + yield dirFromZipNode(archive, child); + } + for (const [name, entry] of node.files) { + yield fileFromZipEntry(archive, name, entry); + } + }, + getFile(path) { + const segments = splitPath(path); + const filename = segments.pop(); + if (!filename) { + return Promise.resolve(null); + } + let dir: ZipNode | null | undefined = node; + for (const segment of segments) { + dir = segment === ".." ? dir.parent : lenientGet(dir.dirs, segment); + if (!dir) { + return Promise.resolve(null); + } + } + const entry = lenientGet(dir.files, filename); + return Promise.resolve( + entry ? fileFromZipEntry(archive, filename, entry) : null, + ); + }, + }; +} + +/** + * Decompressed entries, keyed by the entry they came from. A single image + * routinely gets picked for more than one role in a song, and looking one up + * twice hands back two separate wrappers around the same entry, so without + * this it would be decompressed once per use. + */ +const readEntries = new WeakMap>(); + +/** + * @param archive the source archive + * @param name the entry's own filename + * @param entry the entry to wrap + * @returns the entry as a virtual filesystem file, read lazily + */ +function fileFromZipEntry( + archive: Blob, + name: string, + entry: ZipEntry, +): FileLike { + return { + type: "file", + name, + file() { + let pending = readEntries.get(entry); + if (!pending) { + pending = readEntry(archive, entry).then( + (contents) => new File([contents], name), + ); + readEntries.set(entry, pending); + } + return pending; + }, + }; +} + +/** + * Opens a zip archive as a virtual filesystem directory. Only the archive's + * index is read here; entries are decompressed individually, on demand. + * @param archive the zip file + * @param name a name for the archive's root directory + * @returns the root of the archive + */ +export async function openZip(archive: Blob, name = ""): Promise { + const entries = await readCentralDirectory(archive); + return dirFromZipNode(archive, buildTree(entries, name)); +} + +/** + * @param blob a file that may or may not be a zip archive + * @returns true if the file starts with the zip magic number + */ +export async function isZip(blob: Blob): Promise { + if (blob.size < 4) { + return false; + } + const magic = new Uint8Array(await blob.slice(0, 4).arrayBuffer()); + // "PK\x03\x04", the local file header signature every zip starts with + return ( + magic[0] === 0x50 && + magic[1] === 0x4b && + magic[2] === 0x03 && + magic[3] === 0x04 + ); +} diff --git a/src/browser/zip.ts b/src/browser/zip.ts new file mode 100644 index 0000000..931330b --- /dev/null +++ b/src/browser/zip.ts @@ -0,0 +1,274 @@ +/** + * A minimal, dependency-free reader for zip archives. + * + * Only the central directory is read up front. Individual entries are read + * lazily by slicing the source blob, so opening a multi-gigabyte pack archive + * costs a few kilobytes of reads and only the files actually asked for are + * ever decompressed. + * + * Supports stored (method 0) and deflated (method 8) entries, zip64 archives, + * and both utf-8 and cp437 filename encodings. + */ + +const EOCD_SIG = 0x06054b50; +const ZIP64_EOCD_SIG = 0x06064b50; +const ZIP64_LOCATOR_SIG = 0x07064b50; +const CENTRAL_FILE_SIG = 0x02014b50; +const LOCAL_FILE_SIG = 0x04034b50; + +const EOCD_MIN_SIZE = 22; +/** the comment trailing the EOCD record is length-prefixed with a uint16 */ +const MAX_COMMENT_SIZE = 0xffff; +const ZIP64_LOCATOR_SIZE = 20; + +/** sentinel stored in 32 bit fields whose real value lives in a zip64 extra field */ +const ZIP64_MARKER = 0xffffffff; +const ZIP64_MARKER_16 = 0xffff; + +const STORED = 0; +const DEFLATED = 8; + +/** bit 11 of the general purpose flags, set when the filename is utf-8 */ +const UTF8_FLAG = 0x800; + +/** upper half of code page 437, the encoding of filenames in older archives */ +// prettier-ignore +const CP437_HIGH = + "ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜ¢£¥₧ƒáíóúñѪº¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ "; + +export interface ZipEntry { + /** full path of the entry within the archive, using forward slashes */ + name: string; + /** offset of this entry's local file header within the archive */ + headerOffset: number; + compressedSize: number; + uncompressedSize: number; + /** zip compression method; 0 is stored, 8 is deflated */ + method: number; + isDirectory: boolean; +} + +/** + * Decodes a filename from raw bytes using whichever encoding the entry declares + * @param bytes raw filename bytes + * @param flags the entry's general purpose bit flags + * @returns the decoded filename + */ +function decodeFilename(bytes: Uint8Array, flags: number): string { + if (flags & UTF8_FLAG) { + return new TextDecoder("utf-8").decode(bytes); + } + let result = ""; + for (const byte of bytes) { + result += byte < 0x80 ? String.fromCharCode(byte) : CP437_HIGH[byte - 0x80]; + } + return result; +} + +/** + * Reads a 64 bit little endian integer, which JS can only hold exactly up to + * 2^53. Archives that large are well beyond what a browser could parse anyway. + * @param view a data view over the record + * @param offset byte offset to read from + * @returns the value as a number + */ +function getUint64(view: DataView, offset: number): number { + const value = view.getBigUint64(offset, true); + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error("zip archive is too large to read"); + } + return Number(value); +} + +/** + * @param blob the source archive + * @param start byte offset to read from + * @param end byte offset to read until + * @returns a data view over the requested range + */ +async function readView(blob: Blob, start: number, end: number) { + const clamped = blob.slice(Math.max(0, start), Math.min(blob.size, end)); + return new DataView(await clamped.arrayBuffer()); +} + +interface CentralDirectoryLocation { + offset: number; + size: number; + entryCount: number; +} + +/** + * Scans backwards from the end of the archive for the end of central directory + * record, then follows the zip64 locator if one is present. + * @param blob the source archive + * @returns where the central directory lives and how many entries it holds + */ +async function findCentralDirectory( + blob: Blob, +): Promise { + const tailSize = Math.min(blob.size, EOCD_MIN_SIZE + MAX_COMMENT_SIZE); + const tailStart = blob.size - tailSize; + const tail = await readView(blob, tailStart, blob.size); + + let eocd = -1; + for (let i = tail.byteLength - EOCD_MIN_SIZE; i >= 0; i--) { + if (tail.getUint32(i, true) === EOCD_SIG) { + eocd = i; + break; + } + } + if (eocd === -1) { + throw new Error("not a zip file: no end of central directory record found"); + } + + const location: CentralDirectoryLocation = { + entryCount: tail.getUint16(eocd + 10, true), + size: tail.getUint32(eocd + 12, true), + offset: tail.getUint32(eocd + 16, true), + }; + + const needsZip64 = + location.entryCount === ZIP64_MARKER_16 || + location.size === ZIP64_MARKER || + location.offset === ZIP64_MARKER; + if (!needsZip64) { + return location; + } + + const locator = eocd - ZIP64_LOCATOR_SIZE; + if (locator < 0 || tail.getUint32(locator, true) !== ZIP64_LOCATOR_SIG) { + throw new Error("zip64 archive is missing its end of directory locator"); + } + const zip64Offset = getUint64(tail, locator + 8); + const record = await readView(blob, zip64Offset, zip64Offset + 56); + if (record.getUint32(0, true) !== ZIP64_EOCD_SIG) { + throw new Error("zip64 end of central directory record is corrupt"); + } + return { + entryCount: getUint64(record, 32), + size: getUint64(record, 40), + offset: getUint64(record, 48), + }; +} + +/** + * Pulls the real sizes and offset out of a zip64 extended information extra + * field. Only the fields that overflowed in the base record are present, and + * they always appear in this order. + * @param extra the entry's raw extra field + * @param entry the entry to fill in, mutated in place + */ +function applyZip64Extra(extra: DataView, entry: ZipEntry): void { + let cursor = 0; + while (cursor + 4 <= extra.byteLength) { + const id = extra.getUint16(cursor, true); + const size = extra.getUint16(cursor + 2, true); + const body = cursor + 4; + if (id === 0x0001) { + let field = body; + if (entry.uncompressedSize === ZIP64_MARKER && field + 8 <= body + size) { + entry.uncompressedSize = getUint64(extra, field); + field += 8; + } + if (entry.compressedSize === ZIP64_MARKER && field + 8 <= body + size) { + entry.compressedSize = getUint64(extra, field); + field += 8; + } + if (entry.headerOffset === ZIP64_MARKER && field + 8 <= body + size) { + entry.headerOffset = getUint64(extra, field); + } + return; + } + cursor = body + size; + } +} + +/** + * Reads and parses every central directory record in the archive + * @param blob the source archive + * @returns one entry per file and directory in the archive + */ +export async function readCentralDirectory(blob: Blob): Promise { + const { offset, size, entryCount } = await findCentralDirectory(blob); + const directory = await readView(blob, offset, offset + size); + const bytes = new Uint8Array(directory.buffer); + + const entries: ZipEntry[] = []; + let cursor = 0; + while (entries.length < entryCount && cursor + 46 <= directory.byteLength) { + if (directory.getUint32(cursor, true) !== CENTRAL_FILE_SIG) { + break; + } + const flags = directory.getUint16(cursor + 8, true); + const nameLength = directory.getUint16(cursor + 28, true); + const extraLength = directory.getUint16(cursor + 30, true); + const commentLength = directory.getUint16(cursor + 32, true); + const nameStart = cursor + 46; + + const name = decodeFilename( + bytes.subarray(nameStart, nameStart + nameLength), + flags, + ); + const entry: ZipEntry = { + name, + method: directory.getUint16(cursor + 10, true), + compressedSize: directory.getUint32(cursor + 20, true), + uncompressedSize: directory.getUint32(cursor + 24, true), + headerOffset: directory.getUint32(cursor + 42, true), + isDirectory: name.endsWith("/"), + }; + if (extraLength) { + applyZip64Extra( + new DataView( + directory.buffer, + nameStart + nameLength, + Math.min(extraLength, directory.byteLength - nameStart - nameLength), + ), + entry, + ); + } + entries.push(entry); + cursor = nameStart + nameLength + extraLength + commentLength; + } + return entries; +} + +/** + * Reads a single entry's bytes out of the archive, decompressing if needed. + * + * The local file header has to be re-read here because its extra field is + * frequently a different length than the one in the central directory, so it + * is the only reliable way to find where the entry's data actually starts. + * @param blob the source archive + * @param entry the entry to read + * @returns the entry's decompressed contents + */ +export async function readEntry(blob: Blob, entry: ZipEntry): Promise { + const header = await readView( + blob, + entry.headerOffset, + entry.headerOffset + 30, + ); + if (header.byteLength < 30 || header.getUint32(0, true) !== LOCAL_FILE_SIG) { + throw new Error(`corrupt local file header for '${entry.name}'`); + } + const dataStart = + entry.headerOffset + + 30 + + header.getUint16(26, true) + + header.getUint16(28, true); + const data = blob.slice(dataStart, dataStart + entry.compressedSize); + + if (entry.method === STORED) { + return data; + } + if (entry.method !== DEFLATED) { + throw new Error( + `unsupported compression method ${entry.method} for '${entry.name}'`, + ); + } + const decompressed = data + .stream() + .pipeThrough(new DecompressionStream("deflate-raw")); + return new Response(decompressed).blob(); +} diff --git a/tsconfig.json b/tsconfig.json index 6e6818b..8f08443 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,5 +24,5 @@ "lib": ["ES2022", "DOM"] }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "**/*.test.ts"] + "exclude": ["node_modules", "src/__tests__"] } From 9fac06c66348836e59c8036fff0e4dc158f97d65 Mon Sep 17 00:00:00 2001 From: Noah Manneschmidt Date: Wed, 12 Aug 2026 22:02:40 -0700 Subject: [PATCH 2/3] cleaner package contents --- .npmignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.npmignore b/.npmignore index 24e97ee..d2fe907 100644 --- a/.npmignore +++ b/.npmignore @@ -7,8 +7,10 @@ package-lock.json .editorconfig .nvmrc .yarn +.vscode +.yarnrc.yml Makefile -.eslintrc.cjs +*.config.mjs tsconfig.json *.sh .scripts From c2071a6602a19e0e5ed389034f61c0964607326c Mon Sep 17 00:00:00 2001 From: Noah Manneschmidt Date: Wed, 12 Aug 2026 23:21:04 -0700 Subject: [PATCH 3/3] zips in node, unified api, 1.0.0 beta --- .tool-versions | 2 +- CHANGELOG.md | 15 +- README.md | 51 ++- package.json | 4 +- src/__tests__/calculateStats.test.ts | 6 +- src/__tests__/main.test.ts | 4 +- src/__tests__/node.test.ts | 318 ++++++++++++++ src/__tests__/packFixtures.ts | 66 +++ src/__tests__/parseSimfile.test.ts | 69 +-- .../{browserZip.test.ts => zip.test.ts} | 196 +++------ src/browser/index.ts | 283 +++---------- src/browser/parseSong.ts | 183 -------- src/browser/shared.ts | 12 - src/browser/vfs.ts | 397 ------------------ src/main.ts | 115 +++-- src/parsePack.ts | 195 +++++++++ src/parseSong.ts | 133 +++--- src/types.ts | 26 +- src/util.ts | 13 + src/vfs/archive.ts | 176 ++++++++ src/vfs/dom.ts | 185 ++++++++ src/vfs/index.ts | 96 +++++ src/vfs/node.ts | 139 ++++++ src/{browser => vfs}/zip.ts | 0 24 files changed, 1544 insertions(+), 1140 deletions(-) create mode 100644 src/__tests__/node.test.ts create mode 100644 src/__tests__/packFixtures.ts rename src/__tests__/{browserZip.test.ts => zip.test.ts} (70%) delete mode 100644 src/browser/parseSong.ts delete mode 100644 src/browser/shared.ts delete mode 100644 src/browser/vfs.ts create mode 100644 src/parsePack.ts create mode 100644 src/vfs/archive.ts create mode 100644 src/vfs/dom.ts create mode 100644 src/vfs/index.ts create mode 100644 src/vfs/node.ts rename src/{browser => vfs}/zip.ts (100%) diff --git a/.tool-versions b/.tool-versions index e4e06f8..d0b4db8 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -nodejs 24.12 +nodejs 26 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee24b1..84496ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,17 @@ # Changelog -## v0.10.0 - -- Added support for in-browser parsing of packs directly from a zip file, which is how packs are usually distributed. `parsePack` now accepts a dropped or selected zip in addition to a folder, and the new `parseZipPack` export takes a `File`/`Blob` directly. Archives are read lazily, so only the chart files and images are decompressed and the whole pack never has to be held in memory. Song folders may sit at the root of the archive or inside a pack folder. +## v1.0.0 + +Node and browser now share one API, and every parsing function reads a `.zip` as well as local unpacked files and folders. + +- NEW: Added support for parsing of packs directly from a zip file. `parsePack` now accepts zip file in addition to a folder, and the new `parseZipPack` export takes a `File`/`Blob` directly. Archives are read lazily, so only the chart files and images are decompressed and the whole pack never has to be held in memory. Song folders may sit at the root of the archive or nested inside a pack folder. +- **BREAKING** `parsePack`, `parseAllPacks`, and `parseSong` are now asynchronous and return promises. +- **BREAKING** all three accept a folder **or** a `.zip`, `parsePack` and `parseSong` also accept a `Blob`/`File` directly. `parseAllPacks` walks a `Songs` directory holding any mix of pack folders and archives. The separate `parseZipPack` is gone; use `parsePack`. +- **BREAKING** a song's `banner`, `bg`, and `jacket` are now `ImageRef` handles `{ name, path, file() }` instead of bare strings. `path` is the location on disk, or `null` for an image inside an archive, and `file()` reads the bytes on demand. +- **BREAKING** only images that actually exist are reported. A song tagging an image it doesn't ship now gets `null` instead of a filename pointing at nothing, and an empty tag gets `null` instead of `""`. +- **BREAKING** `Title.titleDir` is now the song's folder name on every platform; the on-disk location moved to the new `Title.titlePath`, which is `null` for songs inside an archive. `Pack.dir` likewise holds the folder name, with the new `Pack.path` holding the location. +- **BREAKING** the browser entry point exports `parseSong` in place of `parseSongFolderOrData`, and the `BrowserSimfile`/`BrowserTitle` are now replaced with plain `Simfile` and `Title`. +- **BREAKING** raised the minimum node version from 16.9 to 20. - Fixed browser directory listings being truncated for large folders. ## v0.9.0 diff --git a/README.md b/README.md index c2d175a..12992b9 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Original parsing code from [city41/stepcharts](https://github.com/city41/stepcha ## Usage ```ts -// in node.js >= 16.9.0 +// in node.js >= 20 import { parseAllPacks, @@ -18,10 +18,12 @@ import { calculateStats, } from "simfile-parser"; -// Use one of the three parsing functions depending on your needs: -const allMyStuff = parseAllPacks("/pathToStepmania/Songs"); -const aGreatPack = parsePack("/pathToStepmania/Songs/DDRMAX2"); -const aGreatSong = parseSong(".../Songs/Easy as Pie 2/Abracadabra"); +// Use one of the three parsing functions depending on your needs. +// Each takes a folder or a .zip, and each returns a promise. +const allMyStuff = await parseAllPacks("/pathToStepmania/Songs"); +const aGreatPack = await parsePack("/pathToStepmania/Songs/DDRMAX2"); +const alsoAPack = await parsePack("/downloads/DDRMAX2.zip"); +const aGreatSong = await parseSong(".../Songs/Easy as Pie 2/Abracadabra"); // you can get some top level info about a song's contents too: calculateStats(aGreatSong.charts["single-challenge"]); @@ -37,8 +39,10 @@ calculateStats(aGreatSong.charts["single-challenge"]); ### Browser support -Support dragging packs directly into a web app by parsing in-browser! A pack -can be either a folder of song folders or a **zip file** containing one. +Support dragging packs directly into a web app by parsing in-browser! The +browser entry point offers the same `parsePack` and `parseSong`, taking +anything the browser hands you: a `DataTransferItem`, an `HTMLInputElement`, a +`File`, or a `Blob`. ```ts // requires typescript 5.0 in "Bundler" module resolution mode for typings @@ -49,7 +53,7 @@ document.body.addEventListener("dragover", function (e) { e.preventDefault(); }); -document.body.addEventListener("drop", async function (e) { +document.body.addEventListener("drop", async function (evt) { // also necessary to prevent browser navigating to dropped folder evt.preventDefault(); if (!evt.dataTransfer) { @@ -70,32 +74,25 @@ document.body.addEventListener("drop", async function (e) { }); ``` -#### Zipped packs - -`parsePack` detects zip files by content, so a pack dropped or selected as an -archive needs no unzipping first. You can also hand one straight to -`parseZipPack`, for example from a file input or a `fetch`: +An archive can also be handed over directly, for example from a `fetch`: ```ts -import { parseZipPack } from "simfile-parser/browser"; - const response = await fetch("/packs/Club Fantastic Season 1.zip"); -const pack = await parseZipPack(await response.blob(), "Club Fantastic"); +const pack = await parsePack(await response.blob(), "Club Fantastic"); ``` -Archives are read lazily: only the archive index, each song's chart file, and -its images are ever decompressed, so the audio and video that make up the bulk -of a pack are skipped entirely and the whole archive never has to be held in -memory. +In the browser `path` is always `null`, since browsers never expose real paths. -Only one pack per archive is supported. An archive holding several packs — a -whole `Songs` directory, say — throws rather than quietly parsing nothing: +### Zipped packs -``` -expected an archive holding a single pack, but found 2: 'DDRMAX2', 'SuperNOVA2' -``` +Every parsing function can read a zip file directly rather than making you unzip it first. Archives are detected by content, not by file extension. `parsePack` also accepts a `Blob` or `File`, so an archive you already have in memory never has to be written out. + +Given a path, an archive is read lazily off disk: only its index and each +song's chart file are read to parse a pack, and images are located but not +loaded until you ask for them. Parsing a 55 MB pack reads **0.44 MB**, and the +archive is never held in memory. -Reading zips uses [`DecompressionStream`][ds], which needs Chrome 103+, -Firefox 113+, or Safari 16.4+. Encrypted archives are not supported. +Reading zips uses [`DecompressionStream`][ds], which needs node 20+, Chrome +103+, Firefox 113+, or Safari 16.4+. Encrypted archives are not supported. [ds]: https://developer.mozilla.org/en-US/docs/Web/API/DecompressionStream diff --git a/package.json b/package.json index 3f52c26..478909e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "simfile-parser", - "version": "0.10.0-beta.0", + "version": "1.0.0-beta.0", "description": "Read stepmania charts with javascript!", "type": "module", "main": "./dist/main.js", @@ -39,7 +39,7 @@ "url": "https://github.com/noahm/simfile-parser/issues" }, "engines": { - "node": ">=16.9.0" + "node": ">=20.0.0" }, "homepage": "https://github.com/noahm/simfile-parser#readme", "devDependencies": { diff --git a/src/__tests__/calculateStats.test.ts b/src/__tests__/calculateStats.test.ts index a277086..4574770 100644 --- a/src/__tests__/calculateStats.test.ts +++ b/src/__tests__/calculateStats.test.ts @@ -5,9 +5,11 @@ import { parseSong, setErrorTolerance } from "../main"; setErrorTolerance("ignore"); const packsRoot = path.resolve(import.meta.dirname, "../../packs"); -test("stats", () => { +test("stats", async () => { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const song = parseSong(path.join(packsRoot, "Easy As Pie 2", "Abracadabra"))!; + const song = await parseSong( + path.join(packsRoot, "Easy As Pie 2", "Abracadabra"), + )!; expect(calculateStats(song.charts["single-challenge"])) .toMatchInlineSnapshot(` { diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 0e4d077..a909dfe 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -6,9 +6,9 @@ setErrorTolerance("bail"); const packsRoot = path.resolve(import.meta.dirname, "../../packs"); describe("parseAllPacks", () => { - test("parses each pack separately", () => { + test("parses each pack separately", async () => { expect( - parseAllPacks(packsRoot).map((p) => ({ + (await parseAllPacks(packsRoot)).map((p) => ({ name: p.name, songs: p.songCount, })), diff --git a/src/__tests__/node.test.ts b/src/__tests__/node.test.ts new file mode 100644 index 0000000..1a74e09 --- /dev/null +++ b/src/__tests__/node.test.ts @@ -0,0 +1,318 @@ +import { randomBytes } from "node:crypto"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { parseAllPacks, parsePack, parseSong } from "../main"; +import { makeZip } from "./makeZip"; +import { found, packsRoot, readPackFiles, zipPack } from "./packFixtures"; +import { setErrorTolerance } from "../util"; + +setErrorTolerance("bail"); + +const fixturePack = "Bhop Ball"; +const fixturePackPath = path.join(packsRoot, fixturePack); + +let tmpDir: string; + +beforeAll(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "simfile-parser-")); +}); + +afterAll(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); +}); + +/** + * Writes an archive out to a real file, since reading one off disk is the + * thing the node entry point adds. + * @param name filename to write to + * @param blob the archive + * @returns the path the archive was written to + */ +async function writeArchive(name: string, blob: Blob): Promise { + const target = path.join(tmpDir, name); + fs.writeFileSync(target, new Uint8Array(await blob.arrayBuffer())); + return target; +} + +/** + * @param pack a parsed pack + * @returns comparable fields for each song, sorted by title + */ +const comparable = (pack: Awaited>) => + pack.simfiles + .map((s) => ({ + title: s.title.titleName, + artist: s.artist, + displayBpm: s.displayBpm, + minBpm: s.minBpm, + maxBpm: s.maxBpm, + charts: Object.keys(s.charts).sort(), + })) + .sort((a, b) => a.title.localeCompare(b.title)); + +describe("parsePack from a folder", () => { + test("parses a pack directory", async () => { + const pack = await parsePack(fixturePackPath); + + expect(pack.name).toBe(fixturePack); + expect(pack.dir).toBe(fixturePack); + expect(pack.path).toBe(fixturePackPath); + expect(pack.songCount).toBe(2); + }); + + test("reports where each song and image lives on disk", async () => { + const pack = await parsePack(fixturePackPath); + const song = found( + pack.simfiles.find((s) => s.title.titleDir.includes("Central Utopia")), + "Central Utopia", + ); + + expect(song.title.titlePath).toBe( + path.join(fixturePackPath, song.title.titleDir), + ); + + const banner = found(song.title.banner, "a banner"); + expect(banner.path).not.toBeNull(); + expect(fs.existsSync(banner.path as string)).toBe(true); + // and the handle reads the very same bytes + expect(new Uint8Array(await (await banner.file()).arrayBuffer())).toEqual( + new Uint8Array(fs.readFileSync(banner.path as string)), + ); + }); + + test("finds a pack wrapped in a download folder", async () => { + const wrapper = path.join(tmpDir, "downloads"); + fs.mkdirSync(path.join(wrapper, fixturePack), { recursive: true }); + fs.cpSync(fixturePackPath, path.join(wrapper, fixturePack), { + recursive: true, + }); + + const pack = await parsePack(wrapper); + + expect(pack.name).toBe(fixturePack); + expect(pack.songCount).toBe(2); + }); + + test("refuses a folder holding several packs", async () => { + // pointed at a whole Songs directory, this used to hand back a pack with + // no songs in it rather than saying anything + await expect(parsePack(packsRoot)).rejects.toThrow( + /expected a single pack, but found 11: .*and 6 more/, + ); + }); + + test("refuses a folder with no songs in it", async () => { + const empty = path.join(tmpDir, "empty"); + fs.mkdirSync(empty, { recursive: true }); + + await expect(parsePack(empty)).rejects.toThrow(/found no songs here/); + }); +}); + +describe("parsePack from a zip", () => { + test("parses a pack from a path on disk", async () => { + const archive = await writeArchive( + "songs-at-root.zip", + await zipPack(fixturePack), + ); + + const pack = await parsePack(archive); + + expect(comparable(pack)).toEqual( + comparable(await parsePack(fixturePackPath)), + ); + }); + + test("matches the same pack parsed from its folder", async () => { + const archive = await writeArchive( + "wrapped.zip", + await zipPack(fixturePack, `${fixturePack}/`), + ); + + const fromZip = await parsePack(archive); + const fromDisk = await parsePack(fixturePackPath); + + expect(fromZip.name).toBe(fromDisk.name); + expect(comparable(fromZip)).toEqual(comparable(fromDisk)); + // the one thing that legitimately differs + expect(fromZip.path).toBeNull(); + expect(fromDisk.path).toBe(fixturePackPath); + }); + + test("names the pack after the file when songs sit at the archive root", async () => { + const archive = await writeArchive( + "Some Cool Pack.zip", + await zipPack(fixturePack), + ); + + expect((await parsePack(archive)).name).toBe("Some Cool Pack"); + }); + + test("accepts an explicit pack name", async () => { + const archive = await writeArchive( + "named.zip", + await zipPack(fixturePack, `${fixturePack}/`), + ); + + expect((await parsePack(archive, "Custom Name")).name).toBe("Custom Name"); + }); + + test("accepts a File without touching the disk", async () => { + const pack = await parsePack(await zipPack(fixturePack)); + + expect(pack.name).toBe(fixturePack); + expect(pack.path).toBeNull(); + }); + + test("accepts a bare Blob", async () => { + const file = await zipPack(fixturePack); + const pack = await parsePack(new Blob([file]), "From A Blob"); + + expect(pack.name).toBe("From A Blob"); + expect(pack.songCount).toBe(2); + }); + + test("returns images as handles with no path but real bytes", async () => { + const archive = await writeArchive( + "images.zip", + await zipPack(fixturePack), + ); + + const pack = await parsePack(archive); + const withBanner = found( + pack.simfiles.find((s) => s.title.banner), + "a song with a banner", + ); + const banner = found(withBanner.title.banner, "the banner"); + + expect(banner.path).toBeNull(); + + // the same image, read straight off disk, should match byte for byte + const onDisk = found( + readPackFiles(fixturePack).find((f) => f.name.endsWith(banner.name)), + `${banner.name} on disk`, + ); + const bytes = new Uint8Array(await (await banner.file()).arrayBuffer()); + expect(bytes).toEqual(onDisk.data); + }); + + test("reads only a small fraction of the archive", async () => { + // stand in for the audio a real pack carries, which is never parsed. It + // has to be incompressible for the archive to stay large, and stored so + // building the fixture stays quick. + const audio = readPackFiles(fixturePack) + .filter((file) => file.name.endsWith(".sm")) + .map((file) => ({ + name: file.name.replace(/\.sm$/, ".ogg"), + data: new Uint8Array(randomBytes(2 * 1024 * 1024)), + })); + const archive = await makeZip([...readPackFiles(fixturePack), ...audio], { + stored: true, + }); + + // every read the zip reader makes goes through Blob.slice, so counting the + // ranges it asks for shows how much of the archive it actually touched + const stats = { bytes: 0 }; + const counting = new Proxy(archive, { + get(target, prop, receiver) { + if (prop === "slice") { + return (start = 0, end = target.size) => { + stats.bytes += Math.min(end, target.size) - Math.max(start, 0); + return target.slice(start, end); + }; + } + const value: unknown = Reflect.get(target, prop, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + + const pack = await parsePack(counting, fixturePack); + + expect(pack.songCount).toBe(2); + // images are only located during parsing, never read, so this is now well + // under even the charts and images the pack holds + const audioBytes = audio.reduce((sum, file) => sum + file.data.length, 0); + expect(audioBytes).toBeGreaterThan(archive.size * 0.8); + expect(stats.bytes).toBeLessThan(archive.size / 4); + }); + + test("rejects a file that isn't a zip", async () => { + const notAZip = path.join(tmpDir, "not-a-zip.txt"); + fs.writeFileSync(notAZip, "just some text"); + + await expect(parsePack(notAZip)).rejects.toThrow(/but got a single file/); + }); + + test("reports a missing file", async () => { + await expect(parsePack(path.join(tmpDir, "nope.zip"))).rejects.toThrow(); + }); + + test("refuses an archive holding more than one pack", async () => { + const archive = await writeArchive( + "two-packs.zip", + await makeZip([ + ...readPackFiles(fixturePack, "Pack A/"), + ...readPackFiles(fixturePack, "Pack B/"), + ]), + ); + + await expect(parsePack(archive)).rejects.toThrow( + "expected a single pack, but found 2: 'Pack A', 'Pack B'", + ); + }); +}); + +describe("parseSong", () => { + test("parses a song folder from a path", async () => { + const song = await parseSong( + path.join(fixturePackPath, "[T10] Central Utopia"), + ); + + expect(song?.title.titleName).toBe("[T10] Central Utopia"); + expect(song?.title.titleDir).toBe("[T10] Central Utopia"); + expect(song?.title.titlePath).toBe( + path.join(fixturePackPath, "[T10] Central Utopia"), + ); + }); + + test("parses a lone chart file from a path", async () => { + const source = found( + readPackFiles(fixturePack).find((f) => f.name.endsWith(".sm")), + "a chart file", + ); + const chart = path.join(tmpDir, "steps.sm"); + fs.writeFileSync(chart, source.data); + + const song = await parseSong(chart); + + expect(song?.title.titleName).toBeTruthy(); + expect(song?.title.titlePath).toBe(chart); + // no folder to look in, so no images + expect(song?.title.banner).toBeNull(); + }); +}); + +describe("parseAllPacks", () => { + test("parses folders and archives side by side", async () => { + const root = path.join(tmpDir, "Songs"); + fs.mkdirSync(root, { recursive: true }); + fs.cpSync(fixturePackPath, path.join(root, "A Folder Pack"), { + recursive: true, + }); + fs.writeFileSync( + path.join(root, "A Zipped Pack.zip"), + new Uint8Array(await (await zipPack(fixturePack)).arrayBuffer()), + ); + // and something that is neither, which should just be skipped + fs.writeFileSync(path.join(root, "notes.txt"), "ignore me"); + + const packs = await parseAllPacks(root); + + expect(packs.map((p) => p.name).sort()).toEqual([ + "A Folder Pack", + "A Zipped Pack", + ]); + expect(packs.every((p) => p.songCount === 2)).toBe(true); + }); +}); diff --git a/src/__tests__/packFixtures.ts b/src/__tests__/packFixtures.ts new file mode 100644 index 0000000..e61ea44 --- /dev/null +++ b/src/__tests__/packFixtures.ts @@ -0,0 +1,66 @@ +/** + * Helpers for building zip fixtures out of the real packs in this repo, shared + * by the browser and node zip test suites. + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { makeZip, ZipFixtureFile, ZipFixtureOptions } from "./makeZip.js"; + +export const packsRoot = path.resolve(import.meta.dirname, "../../packs"); + +/** + * Reads a real pack off disk so it can be zipped up for the end to end tests. + * Audio is skipped to keep the fixtures small; it is never parsed anyway. + * @param packName name of a pack in the packs directory + * @param prefix path to nest the pack's contents under inside the archive + * @returns one entry per file in the pack + */ +export function readPackFiles(packName: string, prefix = ""): ZipFixtureFile[] { + const root = path.join(packsRoot, packName); + const files: ZipFixtureFile[] = []; + const walk = (dir: string) => { + for (const child of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, child.name); + if (child.isDirectory()) { + walk(full); + } else if (!/\.(ogg|mp3|wav|avi|mpg)$/i.test(child.name)) { + files.push({ + name: prefix + path.relative(root, full).split(path.sep).join("/"), + data: new Uint8Array(fs.readFileSync(full)), + }); + } + } + }; + walk(root); + return files; +} + +/** + * @param packName name of a pack in the packs directory + * @param prefix path to nest the pack's contents under inside the archive + * @param options how to encode the archive + * @returns the pack as a zip file + */ +export async function zipPack( + packName: string, + prefix = "", + options: ZipFixtureOptions = {}, +) { + const blob = await makeZip(readPackFiles(packName, prefix), options); + return new File([blob], `${packName}.zip`); +} + +/** + * Asserts something was found, so the tests can go on to use it without + * reaching for non-null assertions. + * @param value the possibly missing value + * @param what a description of what was being looked for + * @returns the value + */ +export function found(value: T | null | undefined, what: string): T { + if (!value) { + throw new Error(`expected to find ${what}`); + } + return value; +} diff --git a/src/__tests__/parseSimfile.test.ts b/src/__tests__/parseSimfile.test.ts index 584983c..e39e01e 100644 --- a/src/__tests__/parseSimfile.test.ts +++ b/src/__tests__/parseSimfile.test.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ import * as path from "path"; -import { parseSong } from "../parseSong"; +import { parseSong } from "../main"; import { Simfile } from "../types"; import { setErrorTolerance } from "../util"; @@ -14,12 +14,20 @@ function scrubDataForSnapshot(simfile: Simfile, assertStepsExist = true) { if (assertStepsExist) expect(chart.arrows).not.toHaveLength(0); chart.arrows = "REDACTED" as any; }); - simfile.title.titleDir = path.relative(packsRoot, simfile.title.titleDir); + simfile.title.titlePath = simfile.title.titlePath + ? path.relative(packsRoot, simfile.title.titlePath) + : null; + // images are lazy handles now; snapshot the filename each one resolved to + for (const role of ["banner", "bg", "jacket"] as const) { + simfile.title[role] = (simfile.title[role]?.name ?? null) as any; + } } describe("parseSong", () => { - test("single old song", () => { - const simfile = parseSong(path.join(packsRoot, "3rdMix", "AFRONOVA"))!; + test("single old song", async () => { + const simfile = await parseSong( + path.join(packsRoot, "3rdMix", "AFRONOVA"), + )!; scrubDataForSnapshot(simfile); expect(simfile).toMatchInlineSnapshot(` { @@ -166,16 +174,17 @@ describe("parseSong", () => { "banner": "AFRONOVA.png", "bg": "AFRONOVA-bg.png", "jacket": null, - "titleDir": "3rdMix/AFRONOVA", + "titleDir": "AFRONOVA", "titleName": "AFRONOVA", + "titlePath": "3rdMix/AFRONOVA", "translitTitleName": null, }, } `); }); - test("single varied bpm song", () => { - const simfile = parseSong( + test("single varied bpm song", async () => { + const simfile = await parseSong( path.join(packsRoot, "A20-(beta)", "Silly Love"), )!; scrubDataForSnapshot(simfile); @@ -1444,18 +1453,19 @@ describe("parseSong", () => { }, "title": { "banner": "Silly Love.png", - "bg": "Silly Love-bg.png", + "bg": "Silly Love.png", "jacket": "Silly Love-jacket.png", - "titleDir": "A20-(beta)/Silly Love", + "titleDir": "Silly Love", "titleName": "Silly Love", + "titlePath": "A20-(beta)/Silly Love", "translitTitleName": null, }, } `); }); - test("single new song", () => { - const simfile = parseSong( + test("single new song", async () => { + const simfile = await parseSong( path.join(packsRoot, "Club Fantastic Season 2", "TerpZone"), )!; scrubDataForSnapshot(simfile); @@ -2811,16 +2821,17 @@ describe("parseSong", () => { "banner": "bn.png", "bg": "bg.png", "jacket": "jacket.png", - "titleDir": "Club Fantastic Season 2/TerpZone", + "titleDir": "TerpZone", "titleName": "TerpZone", + "titlePath": "Club Fantastic Season 2/TerpZone", "translitTitleName": null, }, } `); }); - test("modern varied bpm song", () => { - const simfile = parseSong( + test("modern varied bpm song", async () => { + const simfile = await parseSong( path.join(packsRoot, "BITE6 ITG Customs", "[T10] Neutrino"), )!; scrubDataForSnapshot(simfile); @@ -3131,17 +3142,18 @@ describe("parseSong", () => { "title": { "banner": "bn.png", "bg": "bg.png", - "jacket": "", - "titleDir": "BITE6 ITG Customs/[T10] Neutrino", + "jacket": null, + "titleDir": "[T10] Neutrino", "titleName": "[T10] Neutrino", + "titlePath": "BITE6 ITG Customs/[T10] Neutrino", "translitTitleName": null, }, } `); }); - test("modern display bpm song", () => { - const simfile = parseSong( + test("modern display bpm song", async () => { + const simfile = await parseSong( path.join(packsRoot, "BITE6 ITG Customs", "[T11] Fracture Ray"), )!; scrubDataForSnapshot(simfile); @@ -3565,27 +3577,28 @@ describe("parseSong", () => { "translitSubtitleName": null, }, "title": { - "banner": "fracture-bn.png", - "bg": "fracture-bg.png", - "jacket": "", - "titleDir": "BITE6 ITG Customs/[T11] Fracture Ray", + "banner": null, + "bg": null, + "jacket": null, + "titleDir": "[T11] Fracture Ray", "titleName": "[T11] Fracture Ray", + "titlePath": "BITE6 ITG Customs/[T11] Fracture Ray", "translitTitleName": null, }, } `); }); - test("prefer newer file formats when multiple are available", () => { - const simfile = parseSong( + test("prefer newer file formats when multiple are available", async () => { + const simfile = await parseSong( path.join(packsRoot, "Bhop Ball", "[T07] Ants (No CMOD)"), )!; scrubDataForSnapshot(simfile, false); expect(simfile.title.titleName).toBe("[T07] Ants (No CMOD)"); }); - test("songs with subtitle", () => { - const shoes = parseSong( + test("songs with subtitle", async () => { + const shoes = await parseSong( path.join( packsRoot, "Club Fantastic Season 1", @@ -3598,7 +3611,7 @@ describe("parseSong", () => { translitSubtitleName: null, }); - const bossy = parseSong( + const bossy = await parseSong( path.join( packsRoot, "Club Fantastic Season 2", @@ -3611,7 +3624,7 @@ describe("parseSong", () => { translitSubtitleName: null, }); - const oceania = parseSong( + const oceania = await parseSong( path.join( packsRoot, "Club Fantastic Season 2", diff --git a/src/__tests__/browserZip.test.ts b/src/__tests__/zip.test.ts similarity index 70% rename from src/__tests__/browserZip.test.ts rename to src/__tests__/zip.test.ts index 6402a0d..f1ce3bf 100644 --- a/src/__tests__/browserZip.test.ts +++ b/src/__tests__/zip.test.ts @@ -1,82 +1,25 @@ -import * as fs from "node:fs"; import * as path from "node:path"; import { parsePack as parsePackFromDisk } from "../main"; -import { parsePack, parseZipPack } from "../browser/index"; -import { openZip, isZip, DirLike, isDir } from "../browser/vfs"; -import { readCentralDirectory, readEntry } from "../browser/zip"; +import { parsePack, parseSong } from "../browser/index"; +import { isZip, openZip } from "../vfs/archive"; +import { DirLike, entriesOf, isDir } from "../vfs/index"; +import { readCentralDirectory, readEntry } from "../vfs/zip"; import { makeZip, ZipFixtureFile, ZipFixtureOptions } from "./makeZip"; +import { found, packsRoot, readPackFiles, zipPack } from "./packFixtures"; import { setErrorTolerance } from "../util"; setErrorTolerance("bail"); -const packsRoot = path.resolve(import.meta.dirname, "../../packs"); const fixturePack = "Bhop Ball"; -/** - * Reads a real pack off disk so it can be zipped up for the end to end tests. - * Audio is skipped to keep the fixtures small; it is never parsed anyway. - * @param packName name of a pack in the packs directory - * @param prefix path to nest the pack's contents under inside the archive - * @returns one entry per file in the pack - */ -function readPackFiles(packName: string, prefix = ""): ZipFixtureFile[] { - const root = path.join(packsRoot, packName); - const files: ZipFixtureFile[] = []; - const walk = (dir: string) => { - for (const child of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, child.name); - if (child.isDirectory()) { - walk(full); - } else if (!/\.(ogg|mp3|wav|avi|mpg)$/i.test(child.name)) { - files.push({ - name: prefix + path.relative(root, full).split(path.sep).join("/"), - data: new Uint8Array(fs.readFileSync(full)), - }); - } - } - }; - walk(root); - return files; -} - -/** - * @param packName name of a pack in the packs directory - * @param prefix path to nest the pack's contents under inside the archive - * @param options how to encode the archive - * @returns the pack as a zip file - */ -async function zipPack( - packName: string, - prefix = "", - options: ZipFixtureOptions = {}, -) { - const blob = await makeZip(readPackFiles(packName, prefix), options); - return new File([blob], `${packName}.zip`); -} - -/** - * Asserts something was found, so the tests can go on to use it without - * reaching for non-null assertions. - * @param value the possibly missing value - * @param what a description of what was being looked for - * @returns the value - */ -function found(value: T | null | undefined, what: string): T { - if (!value) { - throw new Error(`expected to find ${what}`); - } - return value; -} - /** * @param dir a virtual directory * @returns the names of its children, sorted */ async function childNames(dir: DirLike) { - const names: string[] = []; - for await (const entry of dir.entries()) { - names.push(isDir(entry) ? `${entry.name}/` : entry.name); - } + const names = (await entriesOf(dir)).map((entry) => + isDir(entry) ? `${entry.name}/` : entry.name, + ); return names.sort(); } @@ -190,7 +133,7 @@ describe("openZip", () => { const root = await openZip(await sampleArchive(), "archive"); expect(await childNames(root)).toEqual(["Pack/"]); - const [pack] = [...(await entriesOf(root))]; + const [pack] = await entriesOf(root); expect(await childNames(pack as DirLike)).toEqual([ "Song One/", "Song Two/", @@ -204,6 +147,15 @@ describe("openZip", () => { expect(await childNames(pack)).not.toContain(".DS_Store"); }); + test("reports no path for anything inside an archive", async () => { + const root = await openZip(await sampleArchive(), "archive"); + const pack = (await entriesOf(root))[0] as DirLike; + expect(pack.path).toBeNull(); + expect( + found(await pack.getFile("Song One/steps.sm"), "steps.sm").path, + ).toBeNull(); + }); + test("resolves paths relative to a directory", async () => { const root = await openZip(await sampleArchive(), "archive"); const pack = (await entriesOf(root))[0] as DirLike; @@ -226,6 +178,8 @@ describe("openZip", () => { // the simfile might tag this as banner.png while the archive has BANNER.png const banner = found(await songOne.getFile("banner.png"), "banner.png"); expect(await (await banner.file()).text()).toBe("banner bytes"); + // and it reports the name the archive really holds, not the one asked for + expect(banner.name).toBe("BANNER.png"); }); test("only reads a given entry once", async () => { @@ -250,42 +204,20 @@ describe("openZip", () => { }); }); -/** - * @param dir a virtual directory - * @returns its children as an array - */ -async function entriesOf(dir: DirLike) { - const all = []; - for await (const entry of dir.entries()) { - all.push(entry); - } - return all; -} - -describe("parseZipPack", () => { +describe("parsePack in the browser", () => { /** - * The node parser reads the same pack straight off disk, so it makes a good - * reference for what the zip parser ought to produce. + * The same pack read straight off disk makes a good reference for what the + * zip path ought to produce. * @returns comparable fields for each song, sorted by title */ - const fromDisk = () => - parsePackFromDisk(path.join(packsRoot, fixturePack)) - .simfiles.map((s) => ({ - title: s.title.titleName, - artist: s.artist, - minBpm: s.minBpm, - maxBpm: s.maxBpm, - displayBpm: s.displayBpm, - stopCount: s.stopCount, - charts: Object.keys(s.charts).sort(), - })) - .sort((a, b) => a.title.localeCompare(b.title)); + const fromDisk = async () => + comparable(await parsePackFromDisk(path.join(packsRoot, fixturePack))); /** * @param pack a parsed pack * @returns comparable fields for each song, sorted by title */ - const comparable = (pack: Awaited>) => + const comparable = (pack: Awaited>) => pack.simfiles .map((s) => ({ title: s.title.titleName, @@ -300,19 +232,19 @@ describe("parseZipPack", () => { test("matches the on-disk parser, for a pack wrapped in a folder", async () => { const zip = await zipPack(fixturePack, `${fixturePack}/`); - const pack = await parseZipPack(zip); + const pack = await parsePack(zip); expect(pack.songCount).toBe(2); expect(pack.name).toBe(fixturePack); - expect(comparable(pack)).toEqual(fromDisk()); + expect(comparable(pack)).toEqual(await fromDisk()); }); test("matches the on-disk parser, for songs at the archive root", async () => { const zip = await zipPack(fixturePack); - const pack = await parseZipPack(zip); + const pack = await parsePack(zip); expect(pack.songCount).toBe(2); // falls back to the archive's own filename for the pack name expect(pack.name).toBe(fixturePack); - expect(comparable(pack)).toEqual(fromDisk()); + expect(comparable(pack)).toEqual(await fromDisk()); }); test("ignores mac metadata sitting alongside the pack folder", async () => { @@ -323,9 +255,9 @@ describe("parseZipPack", () => { { name: "__MACOSX/._" + fixturePack, data: "junk" }, { name: `__MACOSX/${fixturePack}/._steps.sm`, data: "junk" }, ]); - const pack = await parseZipPack(new File([blob], `${fixturePack}.zip`)); + const pack = await parsePack(new File([blob], `${fixturePack}.zip`)); expect(pack.name).toBe(fixturePack); - expect(comparable(pack)).toEqual(fromDisk()); + expect(comparable(pack)).toEqual(await fromDisk()); }); describe("a pack holding only one song", () => { @@ -344,13 +276,13 @@ describe("parseZipPack", () => { // one song folder is the ambiguous case: a lone subfolder is normally a // wrapper to descend through, but here it is the song itself test("finds it at the archive root", async () => { - const pack = await parseZipPack(await singleSongZip("")); + const pack = await parsePack(await singleSongZip("")); expect(pack.songCount).toBe(1); expect(pack.name).toBe("Solo Pack"); }); test("finds it inside a pack folder", async () => { - const pack = await parseZipPack(await singleSongZip("Solo Pack/")); + const pack = await parsePack(await singleSongZip("Solo Pack/")); expect(pack.songCount).toBe(1); expect(pack.name).toBe("Solo Pack"); }); @@ -358,35 +290,36 @@ describe("parseZipPack", () => { test("descends through several wrapper folders", async () => { const zip = await zipPack(fixturePack, `downloads/new/${fixturePack}/`); - const pack = await parseZipPack(zip); + const pack = await parsePack(zip); expect(pack.songCount).toBe(2); expect(pack.name).toBe(fixturePack); }); test("parses stored and zip64 archives the same way", async () => { const reference = comparable( - await parseZipPack(await zipPack(fixturePack, `${fixturePack}/`)), + await parsePack(await zipPack(fixturePack, `${fixturePack}/`)), ); for (const options of [{ stored: true }, { zip64: true }]) { const zip = await zipPack(fixturePack, `${fixturePack}/`, options); - expect(comparable(await parseZipPack(zip))).toEqual(reference); + expect(comparable(await parsePack(zip))).toEqual(reference); } }); - test("extracts images as files", async () => { + test("exposes images as lazy handles carrying the real bytes", async () => { const zip = await zipPack(fixturePack, `${fixturePack}/`); - const pack = await parseZipPack(zip); + const pack = await parsePack(zip); const song = found( pack.simfiles.find((s) => s.title.titleDir.includes("Central Utopia")), "Central Utopia", ); const bg = found(song.title.bg, "a background image"); const banner = found(song.title.banner, "a banner image"); - expect(bg).toBeInstanceOf(File); - expect(banner).toBeInstanceOf(File); + // nothing inside an archive has a location on disk + expect(bg.path).toBeNull(); + expect(banner.path).toBeNull(); // the real image bytes came through, not an empty placeholder - expect(bg.size).toBeGreaterThan(0); - expect(banner.size).toBeGreaterThan(0); + expect((await bg.file()).size).toBeGreaterThan(0); + expect((await banner.file()).size).toBeGreaterThan(0); }); test("still finds the pack when junk folders sit beside it", async () => { @@ -394,9 +327,9 @@ describe("parseZipPack", () => { ...readPackFiles(fixturePack, `${fixturePack}/`), { name: "_screenshots/shot.png", data: "not a song" }, ]); - const pack = await parseZipPack(new File([blob], "download.zip")); + const pack = await parsePack(new File([blob], "download.zip")); expect(pack.name).toBe(fixturePack); - expect(comparable(pack)).toEqual(fromDisk()); + expect(comparable(pack)).toEqual(await fromDisk()); }); describe("archives that aren't a single pack", () => { @@ -413,15 +346,14 @@ describe("parseZipPack", () => { test("refuses an archive holding more than one pack", async () => { const zip = await multiPackZip(["Bhop Ball", "Club Fantastic"]); - await expect(parseZipPack(zip)).rejects.toThrow( - "expected an archive holding a single pack, but found 2: " + - "'Bhop Ball', 'Club Fantastic'", + await expect(parsePack(zip)).rejects.toThrow( + "expected a single pack, but found 2: 'Bhop Ball', 'Club Fantastic'", ); }); test("summarizes the rest when there are lots of packs", async () => { const names = ["A", "B", "C", "D", "E", "F", "G"]; - await expect(parseZipPack(await multiPackZip(names))).rejects.toThrow( + await expect(parsePack(await multiPackZip(names))).rejects.toThrow( "found 7: 'A', 'B', 'C', 'D', 'E', and 2 more", ); }); @@ -430,30 +362,32 @@ describe("parseZipPack", () => { const blob = await makeZip([ { name: "notes/readme.txt", data: "no charts here" }, ]); - await expect(parseZipPack(new File([blob], "notes.zip"))).rejects.toThrow( - /found no songs in this archive/, + await expect(parsePack(new File([blob], "notes.zip"))).rejects.toThrow( + /found no songs here/, ); }); }); test("accepts an explicit pack name", async () => { const zip = await zipPack(fixturePack, `${fixturePack}/`); - expect((await parseZipPack(zip, "Custom Name")).name).toBe("Custom Name"); - }); -}); - -describe("parsePack", () => { - test("accepts a zip file directly", async () => { - const zip = await zipPack(fixturePack, `${fixturePack}/`); - const pack = await parsePack(zip); - expect(pack.songCount).toBe(2); - expect(pack.name).toBe(fixturePack); + expect((await parsePack(zip, "Custom Name")).name).toBe("Custom Name"); }); test("rejects a file that is not a zip or a folder", async () => { const notAPack = new File(["#TITLE:lonely;"], "steps.sm"); - await expect(parsePack(notAPack)).rejects.toThrow( - /expected a folder or zip file/, + await expect(parsePack(notAPack)).rejects.toThrow(/but got a single file/); + }); + + test("parses a lone chart file through parseSong", async () => { + const chart = found( + readPackFiles(fixturePack).find((f) => f.name.endsWith(".sm")), + "a chart file", ); + const song = await parseSong(new File([chart.data], "steps.sm")); + + expect(song?.title.titleName).toBeTruthy(); + // no folder to look in, so no images and nowhere on disk to point at + expect(song?.title.banner).toBeNull(); + expect(song?.title.titlePath).toBeNull(); }); }); diff --git a/src/browser/index.ts b/src/browser/index.ts index 4d010f8..a78f1ab 100644 --- a/src/browser/index.ts +++ b/src/browser/index.ts @@ -1,8 +1,9 @@ -import { supportedExtensions } from "../parsers/index.js"; -import { Pack } from "../types.js"; -import { reportError } from "../util.js"; -import { BrowserSimfile, parseSong } from "./parseSong.js"; -import { AnyEntry, DirLike, fromDom, isDir, isZip, openZip } from "./vfs.js"; +import { parsePackFromEntry, PackWithSongs } from "../parsePack.js"; +import { parseSongFromEntry } from "../parseSong.js"; +import { Simfile } from "../types.js"; +import { isZip, openZip, stripZipExtension } from "../vfs/archive.js"; +import { AnyEntry, isDir, isEntry } from "../vfs/index.js"; +import { fromDom } from "../vfs/dom.js"; declare global { interface DataTransferItem { @@ -11,21 +12,35 @@ declare global { } } -export type PackWithSongs = Pack & { simfiles: BrowserSimfile[] }; +export * from "../types.js"; +export * from "../calculateStats.js"; +export { setErrorTolerance } from "../util.js"; +export type { PackWithSongs } from "../parsePack.js"; +export type { AnyEntry, DirLike, FileLike } from "../vfs/index.js"; -export type { BrowserSimfile, BrowserTitle } from "./parseSong.js"; +/** anything a browser might hand us for a dropped or selected item */ +export type BrowserSource = + | DataTransferItem + | HTMLInputElement + | File + | Blob + | AnyEntry; /** * Pulls a usable file/folder reference out of whatever the browser handed us. - * @param item a dropped item, a file input, or a file + * @param item a dropped item, a file input, a file, or an archive's contents * @returns the item as a virtual filesystem entry */ -async function resolveItem( - item: DataTransferItem | HTMLInputElement | File, -): Promise { +async function resolveItem(item: BrowserSource): Promise { + if (isEntry(item)) { + return item; + } if (item instanceof File) { return fromDom(item); } + if (item instanceof Blob) { + return openZip(item); + } if (item instanceof HTMLInputElement) { if ("webkitEntries" in item && item.webkitEntries.length) { if (item.webkitEntries.length > 1) { @@ -62,237 +77,49 @@ async function resolveItem( } /** - * @param dir a directory to inspect - * @returns true if the directory directly contains a simfile - */ -async function containsSimfile(dir: DirLike): Promise { - for await (const entry of dir.entries()) { - if ( - !isDir(entry) && - supportedExtensions.some((ext) => entry.name.endsWith(ext)) - ) { - return true; - } - } - return false; -} - -/** - * @param dir a directory to inspect - * @returns the directory's immediate subfolders - */ -async function subdirectories(dir: DirLike): Promise { - const subdirs: DirLike[] = []; - for await (const entry of dir.entries()) { - if (isDir(entry)) { - subdirs.push(entry); - } - } - return subdirs; -} - -/** - * @param dir a directory to inspect - * @returns true if any of the directory's subfolders is a song folder - */ -async function looksLikePack(dir: DirLike): Promise { - for (const subdir of await subdirectories(dir)) { - if (await containsSimfile(subdir)) { - return true; - } - } - return false; -} - -/** how many nested wrapper folders to look through before giving up */ -const maxPackDepth = 4; - -type PackSearch = - | { type: "found"; dir: DirLike } - | { type: "multiple"; packs: DirLike[] } - | { type: "none" }; - -/** - * Finds the folder that actually holds the song folders. Archives commonly - * wrap a pack in one or more extra folders, so descend through them until we - * reach a folder whose children look like songs. - * @param dir the root of the archive - * @param depth how many levels have been descended so far - * @returns the pack folder, or why one couldn't be settled on - */ -async function findPackRoot(dir: DirLike, depth = 0): Promise { - if (await looksLikePack(dir)) { - return { type: "found", dir }; - } - - // nothing here is a song, so look for the pack among the subfolders. Doing - // it by what they contain rather than by counting them means junk folders - // sitting next to the pack don't make it ambiguous. - const subdirs = await subdirectories(dir); - const packs: DirLike[] = []; - for (const subdir of subdirs) { - if (await looksLikePack(subdir)) { - packs.push(subdir); - } - } - if (packs.length === 1) { - return { type: "found", dir: packs[0] }; - } - if (packs.length > 1) { - return { type: "multiple", packs }; - } - - if (subdirs.length === 1 && depth < maxPackDepth) { - return findPackRoot(subdirs[0], depth + 1); - } - return { type: "none" }; -} - -/** how many pack names to name individually before summarizing the rest */ -const maxNamesInError = 5; - -/** - * @param packs the packs found in an archive - * @returns an error explaining that only one pack can be parsed at a time - */ -function multiplePacksError(packs: DirLike[]): Error { - // sorted so the message doesn't depend on the order the archive happens to - // list its entries in - const names = packs.map((pack) => `'${pack.name}'`).sort(); - const listed = names.slice(0, maxNamesInError).join(", "); - const rest = names.length - maxNamesInError; - return new Error( - `expected an archive holding a single pack, but found ${names.length}: ` + - (rest > 0 ? `${listed}, and ${rest} more` : listed), - ); -} - -/** - * @param dirName the name of the folder a pack was found in - * @returns pack metadata derived from that folder name - */ -function packFromDirName(dirName: string): Pack { - return { - name: dirName.replace(/-/g, " "), - dir: dirName, - songCount: 0, - }; -} - -/** - * Parses every song folder inside a directory into a pack - * @param dir the pack's folder - * @param pack metadata for the pack being built, mutated with the song count - * @returns parsed pack + * Expands a dropped or selected item into something parsable, opening it as an + * archive if that is what it turns out to be. + * @param item whatever the browser handed us + * @returns the item as a virtual filesystem entry */ -async function parsePackDir(dir: DirLike, pack: Pack): Promise { - const songFolders: DirLike[] = []; - for await (const entry of dir.entries()) { - if (isDir(entry)) { - songFolders.push(entry); - } +async function resolveSource(item: BrowserSource): Promise { + const entry = await resolveItem(item); + if (isDir(entry)) { + return entry; } - - const simfiles: BrowserSimfile[] = []; - for (const songFolder of songFolders) { - try { - const songData = await parseSong(songFolder); - if (songData) { - simfiles.push({ - ...songData, - pack, - }); - } - } catch (e) { - reportError(`parseStepchart failed for '${songFolder.name}'`, e); - } + const file = await entry.file(); + if (await isZip(file)) { + return openZip(file, stripZipExtension(entry.name)); } - - pack.songCount = simfiles.length; - - return { - ...pack, - simfiles, - }; -} - -/** - * @param filename name of a zip file - * @returns the name with any `.zip` extension removed - */ -function stripZipExtension(filename: string) { - return filename.replace(/\.zip$/i, ""); + return entry; } /** - * Parse a pack directly from a zip archive, without unzipping it first. + * Parse a pack drag/dropped or selected by a user in a browser. The pack may + * be a folder of song folders or a `.zip` archive holding one, which is how + * packs are usually distributed; archives are read lazily, so only chart files + * and images are ever decompressed. * - * Only the archive's index and the files belonging to each song are read, so - * large packs don't have to be held in memory all at once. - * @param archive the zip file - * @param name optional pack name. Defaults to the name of the folder the songs - * were found in, falling back to the archive's own filename. - * @throws {Error} if the archive holds more than one pack, or no songs at all - * @returns parsed pack - */ -export async function parseZipPack( - archive: File | Blob, - name?: string, -): Promise { - const archiveName = - archive instanceof File ? stripZipExtension(archive.name) : ""; - const root = await openZip(archive, archiveName); - - const search = await findPackRoot(root); - if (search.type === "multiple") { - throw multiplePacksError(search.packs); - } - if (search.type === "none") { - throw new Error( - "found no songs in this archive; expected a pack containing one folder per song", - ); - } - const packDir = search.dir; - const dirName = packDir.name || archiveName; - return parsePackDir( - packDir, - // an explicitly provided name is used as given, rather than being run - // through the guesswork we apply to folder names - name ? { name, dir: dirName, songCount: 0 } : packFromDirName(dirName), - ); -} - -/** - * Parse a pack drag/dropped by a user in a browser. The pack may be either a - * folder of song folders or a zip archive containing one. + * If the pack is wrapped in extra folders — as archives commonly are — it is + * found inside them. * @param item a DataTransferItem from a drop event, a file input, or a file + * @param name optional pack name, overriding the guess made from the folder + * @throws {Error} if more than one pack is found, or no songs at all * @returns parsed pack */ export async function parsePack( - item: DataTransferItem | HTMLInputElement | File, + item: BrowserSource, + name?: string, ): Promise { - const entry = await resolveItem(item); - - if (!isDir(entry)) { - const file = await entry.file(); - if (!(await isZip(file))) { - throw new Error("expected a folder or zip file, but got another file"); - } - // let parseZipPack name the pack after the folder it finds the songs in, - // falling back to the archive's filename - return parseZipPack(file); - } - - return parsePackDir(entry, packFromDirName(entry.name)); + return parsePackFromEntry(await resolveSource(item), name); } /** - * For parsing a single song instead. Parses either a whole song folder, or just the metadata from a single simfile (ssc/sm/dwi) - * @param item a data transfer item or HTML Input element a user has added a file selection to - * @returns a simfile or null + * Parse a single song, either a whole song folder or just the metadata from an + * individual chart file (ssc/sm/dwi). + * @param item a data transfer item, file input, or file + * @returns a simfile object without pack info, or null if no chart was found */ -export async function parseSongFolderOrData( - item: DataTransferItem | HTMLInputElement | File, -): Promise { - return parseSong(await resolveItem(item)); +export async function parseSong(item: BrowserSource): Promise { + return parseSongFromEntry(await resolveSource(item)); } diff --git a/src/browser/parseSong.ts b/src/browser/parseSong.ts deleted file mode 100644 index f00f323..0000000 --- a/src/browser/parseSong.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { - parsers, - supportedExtensions, - sortFileCandidatesByPriority, -} from "../parsers/index.js"; -import { ParsedImages, RawSimfile } from "../parsers/types.js"; -import { Simfile, Title } from "../types.js"; -import { extname } from "./shared.js"; -import { AnyEntry, DirLike, FileLike, isDir } from "./vfs.js"; - -/** - * Find the best simfile in a given directory - * @param songDir directory to search - * @returns the most preferred simfile found, or null - */ -async function identifySongFile(songDir: DirLike): Promise { - const candidates: FileLike[] = []; - for await (const entry of songDir.entries()) { - if ( - !isDir(entry) && - supportedExtensions.some((ext) => entry.name.endsWith(ext)) - ) { - candidates.push(entry); - } - } - if (!candidates.length) { - return null; - } - candidates.sort((a, b) => sortFileCandidatesByPriority(a.name, b.name)); - return candidates[0]; -} - -const imageExts = new Set([".png", ".jpg"]); - -/** - * Get all image files in a given directory - * @param songDir directory to search - * @yields {FileLike} each file with a supported image extension - */ -async function* getImages(songDir: DirLike) { - for await (const entry of songDir.entries()) { - if (isDir(entry)) { - continue; - } - const ext = extname(entry.name); - if (ext && imageExts.has(ext)) { - yield entry; - } - } -} - -/** - * Make some best guesses about which images should be used for which fields - * @param songDir the song's directory - * @param tagged image metadata found in simfile - * @returns final image metadata - */ -async function guessImages(songDir: DirLike, tagged: ParsedImages) { - let jacket = tagged.jacket ? await songDir.getFile(tagged.jacket) : null; - let bg = tagged.bg ? await songDir.getFile(tagged.bg) : null; - let banner = tagged.banner ? await songDir.getFile(tagged.banner) : null; - const leftovers: FileLike[] = []; - for await (const image of getImages(songDir)) { - const imageName = image.name; - const ext = extname(imageName) || ""; - if ( - (!tagged.jacket && imageName.endsWith("-jacket" + ext)) || - imageName.startsWith("jacket.") - ) { - jacket = image; - } else if ( - (!tagged.bg && imageName.endsWith("-bg" + ext)) || - imageName.startsWith("bg.") - ) { - bg = image; - } else if ( - (!tagged.bg && imageName.endsWith("-bn" + ext)) || - imageName.startsWith("bn.") - ) { - banner = image; - } else { - leftovers.push(image); - } - } - if (!bg && leftovers.length) { - bg = leftovers.shift() || null; - } - if (!banner && leftovers.length) { - banner = leftovers.shift() || null; - } - if (!jacket && leftovers.length) { - jacket = leftovers.shift() || null; - } - return { - jacket: jacket ? await jacket.file() : null, - bg: bg ? await bg.file() : null, - banner: banner ? await banner.file() : null, - }; -} - -/** - * get individual bpms of each chart - * @param sm simfile - * @returns list of found bpms, one per chart - */ -function getBpms(sm: Pick): number[] { - const chart = Object.values(sm.charts)[0]; - return chart.bpm.map((b) => b.bpm); -} - -export type BrowserTitle = Omit & { - banner: File | null; - bg: File | null; - jacket: File | null; -}; - -export type BrowserSimfile = Omit & { - title: BrowserTitle; -}; - -/** - * Parse a single simfile by folder or individual file. Automatically determines which parser to use depending on chart definition type. - * @param songDirOrFile song folder or file reference (contains a chart definition file [dwi/sm/ssc], images, etc) - * @returns a simfile object without mix info or null if no sm/ssc file was found - */ -export async function parseSong( - songDirOrFile: AnyEntry, -): Promise { - const songDir = isDir(songDirOrFile) ? songDirOrFile : null; - const songFile = songDir - ? await identifySongFile(songDir) - : (songDirOrFile as FileLike); - if (!songFile) return null; - - const file = await songFile.file(); - const extension = extname(file.name); - if (!extension) return null; - - const parser = parsers[extension]; - - if (!parser) { - throw new Error(`No parser registered for extension: ${extension}`); - } - - const { images, ...rawStepchart } = parser(await file.text(), ""); - - if (!Object.keys(rawStepchart.charts).length) { - throw new Error( - `Failed to parse any charts from song: ${rawStepchart.title}`, - ); - } - - const bpms = getBpms(rawStepchart); - const minBpm = Math.round(Math.min(...bpms)); - const maxBpm = Math.round(Math.max(...bpms)); - - let displayBpm = rawStepchart.displayBpm; - if (!displayBpm) { - displayBpm = minBpm === maxBpm ? minBpm.toString() : `${minBpm}-${maxBpm}`; - } - - const finalImages = songDir - ? await guessImages(songDir, images) - : { banner: null, bg: null, jacket: null }; - - return { - ...rawStepchart, - title: { - titleName: rawStepchart.title, - translitTitleName: rawStepchart.titletranslit ?? null, - titleDir: songDirOrFile.name, - ...finalImages, - }, - subtitle: { - subtitleName: rawStepchart.subtitle ?? "", - translitSubtitleName: rawStepchart.subtitletranslit ?? null, - }, - minBpm, - maxBpm, - displayBpm, - stopCount: Object.values(rawStepchart.charts)[0].stops.length, - }; -} diff --git a/src/browser/shared.ts b/src/browser/shared.ts deleted file mode 100644 index aede96c..0000000 --- a/src/browser/shared.ts +++ /dev/null @@ -1,12 +0,0 @@ -/** - * returns extension name from a filename - * @param name filename - * @returns extension, with leading period - */ -export function extname(name: string) { - const match = name.match(/.+(\.[^.]+)$/); - if (match) { - return match[1]; - } - return null; -} diff --git a/src/browser/vfs.ts b/src/browser/vfs.ts deleted file mode 100644 index adcdaee..0000000 --- a/src/browser/vfs.ts +++ /dev/null @@ -1,397 +0,0 @@ -/** - * A tiny virtual filesystem the parsers can target, so they don't have to care - * whether a song came from a `FileSystemDirectoryHandle`, the older - * `FileSystemDirectoryEntry`, or a folder inside a zip archive. - */ - -import { readCentralDirectory, readEntry, ZipEntry } from "./zip.js"; - -export interface FileLike { - type: "file"; - name: string; - /** reads the file's contents */ - file(): Promise; -} - -export interface DirLike { - type: "directory"; - name: string; - /** iterates the directory's immediate children */ - entries(): AsyncIterable; - /** - * resolves a path relative to this directory, which may include `..` - * segments, to a file. Resolves to null if it can't be found. - */ - getFile(path: string): Promise; -} - -export type AnyEntry = FileLike | DirLike; - -/** - * @param entry any virtual filesystem entry - * @returns true if the entry is a directory - */ -export function isDir(entry: AnyEntry): entry is DirLike { - return entry.type === "directory"; -} - -/** - * Splits a simfile-relative path into segments, tolerating the backslashes - * that occasionally show up in tags authored on Windows. - * @param path a relative path - * @returns the meaningful path segments - */ -function splitPath(path: string): string[] { - return path.split(/[/\\]/).filter((segment) => segment && segment !== "."); -} - -// --- File System Access API (handles) --------------------------------------- - -/** - * @param handle a file handle - * @returns the handle as a virtual filesystem file - */ -function fileFromHandle(handle: FileSystemFileHandle): FileLike { - return { - type: "file", - name: handle.name, - file: () => handle.getFile(), - }; -} - -/** - * @param handle a directory handle - * @returns the handle as a virtual filesystem directory - */ -function dirFromHandle(handle: FileSystemDirectoryHandle): DirLike { - return { - type: "directory", - name: handle.name, - async *entries() { - for await (const child of handle.values()) { - yield fromHandle(child); - } - }, - async getFile(path) { - const segments = splitPath(path); - const filename = segments.pop(); - if (!filename) { - return null; - } - try { - let dir = handle; - for (const segment of segments) { - if (segment === "..") { - // this api gives no way to walk up out of the granted directory - return null; - } - dir = await dir.getDirectoryHandle(segment); - } - return fileFromHandle(await dir.getFileHandle(filename)); - } catch { - return null; - } - }, - }; -} - -/** - * @param handle any file system handle - * @returns the handle as a virtual filesystem entry - */ -export function fromHandle(handle: FileSystemHandle): AnyEntry { - return handle.kind === "directory" - ? dirFromHandle(handle as FileSystemDirectoryHandle) - : fileFromHandle(handle as FileSystemFileHandle); -} - -// --- legacy drag & drop entries --------------------------------------------- - -/** - * @param entry a file entry - * @returns the entry as a virtual filesystem file - */ -function fileFromEntry(entry: FileSystemFileEntry): FileLike { - return { - type: "file", - name: entry.name, - file: () => new Promise((resolve, reject) => entry.file(resolve, reject)), - }; -} - -/** - * `readEntries` only returns a limited number of children per call, so it has - * to be called until it comes back empty to see a whole directory. - * @param dir a directory entry - * @returns every child of the directory - */ -function readAllEntries(dir: FileSystemDirectoryEntry) { - const reader = dir.createReader(); - const all: FileSystemEntry[] = []; - return new Promise((resolve, reject) => { - const readBatch = () => - reader.readEntries((batch) => { - if (!batch.length) { - resolve(all); - return; - } - all.push(...batch); - readBatch(); - }, reject); - readBatch(); - }); -} - -/** - * @param entry a directory entry - * @returns the entry as a virtual filesystem directory - */ -function dirFromEntry(entry: FileSystemDirectoryEntry): DirLike { - return { - type: "directory", - name: entry.name, - async *entries() { - for (const child of await readAllEntries(entry)) { - yield fromEntry(child); - } - }, - async getFile(path) { - const segments = splitPath(path); - try { - let dir = entry; - while (segments[0] === "..") { - segments.shift(); - dir = await new Promise((resolve, reject) => - dir.getParent(resolve as never, reject), - ); - } - if (!segments.length) { - return null; - } - const found = await new Promise((resolve, reject) => - dir.getFile(segments.join("/"), {}, resolve, reject), - ); - return found.isFile - ? fileFromEntry(found as FileSystemFileEntry) - : null; - } catch { - return null; - } - }, - }; -} - -/** - * @param entry any file system entry - * @returns the entry as a virtual filesystem entry - */ -export function fromEntry(entry: FileSystemEntry): AnyEntry { - return entry.isDirectory - ? dirFromEntry(entry as FileSystemDirectoryEntry) - : fileFromEntry(entry as FileSystemFileEntry); -} - -// --- plain files ------------------------------------------------------------ - -/** - * @param file a file - * @returns the file as a virtual filesystem file - */ -export function fromFile(file: File): FileLike { - return { - type: "file", - name: file.name, - file: () => Promise.resolve(file), - }; -} - -/** - * @param source anything a browser might hand us for a dropped item - * @returns the item as a virtual filesystem entry - */ -export function fromDom( - source: FileSystemHandle | FileSystemEntry | File, -): AnyEntry { - if (source instanceof File) { - return fromFile(source); - } - return "kind" in source ? fromHandle(source) : fromEntry(source); -} - -// --- zip archives ----------------------------------------------------------- - -/** folders some archivers add alongside the real contents */ -const ignoredNames = new Set(["__MACOSX", ".DS_Store", "Thumbs.db"]); - -interface ZipNode { - name: string; - dirs: Map; - files: Map; - parent: ZipNode | null; -} - -/** - * @param name the node's own name - * @param parent the containing node, if any - * @returns an empty tree node - */ -function makeNode(name: string, parent: ZipNode | null): ZipNode { - return { name, dirs: new Map(), files: new Map(), parent }; -} - -/** - * Rebuilds the archive's folder hierarchy from its flat list of entries. - * Intermediate folders are created as needed, since archives are not required - * to include explicit entries for them. - * @param entries every entry in the archive - * @param rootName a name to give the root of the tree - * @returns the root node of the tree - */ -function buildTree(entries: ZipEntry[], rootName: string): ZipNode { - const root = makeNode(rootName, null); - for (const entry of entries) { - const segments = splitPath(entry.name); - if (!segments.length || segments.some((s) => ignoredNames.has(s))) { - continue; - } - const filename = entry.isDirectory ? null : segments.pop(); - let node = root; - for (const segment of segments) { - let child = node.dirs.get(segment); - if (!child) { - child = makeNode(segment, node); - node.dirs.set(segment, child); - } - node = child; - } - if (filename && !ignoredNames.has(filename)) { - node.files.set(filename, entry); - } - } - return root; -} - -/** - * Looks up a key in a map, falling back to a case insensitive match. Packs are - * routinely authored on case insensitive filesystems, so a simfile's tags may - * disagree with the archive on the casing of a filename. - * @param map the map to search - * @param key the key to look for - * @returns the matching value, or undefined - */ -function lenientGet(map: Map, key: string): T | undefined { - const exact = map.get(key); - if (exact !== undefined) { - return exact; - } - const lowered = key.toLowerCase(); - for (const [candidate, value] of map) { - if (candidate.toLowerCase() === lowered) { - return value; - } - } - return undefined; -} - -/** - * @param archive the source archive - * @param node the tree node to wrap - * @returns the node as a virtual filesystem directory - */ -function dirFromZipNode(archive: Blob, node: ZipNode): DirLike { - return { - type: "directory", - name: node.name, - async *entries() { - for (const child of node.dirs.values()) { - yield dirFromZipNode(archive, child); - } - for (const [name, entry] of node.files) { - yield fileFromZipEntry(archive, name, entry); - } - }, - getFile(path) { - const segments = splitPath(path); - const filename = segments.pop(); - if (!filename) { - return Promise.resolve(null); - } - let dir: ZipNode | null | undefined = node; - for (const segment of segments) { - dir = segment === ".." ? dir.parent : lenientGet(dir.dirs, segment); - if (!dir) { - return Promise.resolve(null); - } - } - const entry = lenientGet(dir.files, filename); - return Promise.resolve( - entry ? fileFromZipEntry(archive, filename, entry) : null, - ); - }, - }; -} - -/** - * Decompressed entries, keyed by the entry they came from. A single image - * routinely gets picked for more than one role in a song, and looking one up - * twice hands back two separate wrappers around the same entry, so without - * this it would be decompressed once per use. - */ -const readEntries = new WeakMap>(); - -/** - * @param archive the source archive - * @param name the entry's own filename - * @param entry the entry to wrap - * @returns the entry as a virtual filesystem file, read lazily - */ -function fileFromZipEntry( - archive: Blob, - name: string, - entry: ZipEntry, -): FileLike { - return { - type: "file", - name, - file() { - let pending = readEntries.get(entry); - if (!pending) { - pending = readEntry(archive, entry).then( - (contents) => new File([contents], name), - ); - readEntries.set(entry, pending); - } - return pending; - }, - }; -} - -/** - * Opens a zip archive as a virtual filesystem directory. Only the archive's - * index is read here; entries are decompressed individually, on demand. - * @param archive the zip file - * @param name a name for the archive's root directory - * @returns the root of the archive - */ -export async function openZip(archive: Blob, name = ""): Promise { - const entries = await readCentralDirectory(archive); - return dirFromZipNode(archive, buildTree(entries, name)); -} - -/** - * @param blob a file that may or may not be a zip archive - * @returns true if the file starts with the zip magic number - */ -export async function isZip(blob: Blob): Promise { - if (blob.size < 4) { - return false; - } - const magic = new Uint8Array(await blob.slice(0, 4).arrayBuffer()); - // "PK\x03\x04", the local file header signature every zip starts with - return ( - magic[0] === 0x50 && - magic[1] === 0x4b && - magic[2] === 0x03 && - magic[3] === 0x04 - ); -} diff --git a/src/main.ts b/src/main.ts index 6030f1a..ed479ff 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,87 +1,72 @@ -import * as fs from "node:fs"; import * as path from "node:path"; -import { parseSong } from "./parseSong.js"; -import { Pack, Simfile } from "./types.js"; +import { + packFromDir, + parsePackDir, + parsePackFromEntry, + PackWithSongs, +} from "./parsePack.js"; +import { parseSongFromEntry } from "./parseSong.js"; +import { Simfile } from "./types.js"; import { reportError } from "./util.js"; +import { entriesOf, isDir } from "./vfs/index.js"; +import { dirFromPath, resolveSource, Source } from "./vfs/node.js"; export * from "./types.js"; -export * from "./parseSong.js"; export * from "./calculateStats.js"; export { setErrorTolerance } from "./util.js"; - -export type PackWithSongs = Pack & { - simfiles: Simfile[]; -}; +export type { PackWithSongs } from "./parsePack.js"; +export type { Source } from "./vfs/node.js"; +export type { AnyEntry, DirLike, FileLike } from "./vfs/index.js"; /** - * @param dirPath path segments - * @returns a list of all child entries of the given path + * Parse an entire pack. The pack may be a folder of song folders or a `.zip` + * archive holding one, which is how packs are usually distributed; archives + * are read lazily, so only chart files and images are ever decompressed. + * + * If the pack is wrapped in extra folders — as archives and downloads commonly + * are — it is found inside them. + * @param source path to a pack folder or `.zip`, or an archive's contents + * @param name optional pack name, overriding the guess made from the folder + * @throws {Error} if more than one pack is found, or no songs at all + * @returns info about the pack as a whole and parsed simfiles for each song */ -function getFiles(...dirPath: string[]): string[] { - const builtPath = dirPath.reduce((building, d) => { - return path.join(building, d); - }, ""); - - return fs.readdirSync(builtPath); +export async function parsePack( + source: Source, + name?: string, +): Promise { + return parsePackFromEntry(await resolveSource(source), name); } /** - * @param dirPath path segments - * @returns a list of child directories of the given path + * Convenience function to call {@link parsePack} on everything in a stepmania + * `Songs` directory. Both pack folders and `.zip` archives are picked up, and + * anything that turns out not to hold a pack is skipped. + * @param rootDir path to a directory containing packs + * @returns a list of packs, each with a list of simfiles */ -function getDirectories(...dirPath: string[]): string[] { - const builtPath = dirPath.reduce((building, d) => { - return path.join(building, d); - }, ""); - - return getFiles(builtPath).filter((d) => { - return fs.statSync(path.join(builtPath, d)).isDirectory(); - }); -} - -/** - * Parse an entire pack and return all data - * @param dir path to a pack of songs (contains one or more folders, each containing a song) - * @returns info about the pack as a whole and parsed simfile objects for each song - */ -export function parsePack(dir: string): PackWithSongs { - const songDirs = getDirectories(dir); - - const pack: Pack = { - name: path.basename(dir).replace(/-/g, " "), - dir, - songCount: 0, - }; - - const simfiles: Simfile[] = []; - songDirs.map((songFolder) => { - const songDirPath = path.join(dir, songFolder); +export async function parseAllPacks(rootDir: string): Promise { + const packs: PackWithSongs[] = []; + for (const entry of await entriesOf(dirFromPath(rootDir))) { try { - const songData = parseSong(songDirPath); - if (songData) { - simfiles.push({ - ...songData, - pack, - }); + if (isDir(entry)) { + // each child is taken to be a pack, since the caller already said so + packs.push(await parsePackDir(entry, packFromDir(entry))); + } else if (path.extname(entry.name).toLowerCase() === ".zip") { + packs.push(await parsePack(entry.path ?? "")); } } catch (e) { - reportError(`parseStepchart failed for '${songFolder}'`, e); + reportError(`failed to parse pack '${entry.name}'`, e); } - }); - - pack.songCount = simfiles.length; - - return { - ...pack, - simfiles, - }; + } + return packs; } /** - * Convenience function to call `getPack` on every immediate subdirectory - * @param rootDir path to a stepmania songs directory (contains folders per pack of songs) - * @returns a list of pack objects, each with a list of simfile objects + * Parse a single song, either a whole song folder or just the metadata from an + * individual chart file (ssc/sm/dwi). + * @param source path to a song folder or chart file, or its contents + * @returns a simfile object without pack info, or null if no chart was found */ -export function parseAllPacks(rootDir: string): PackWithSongs[] { - return getDirectories(rootDir).map((d) => parsePack(path.join(rootDir, d))); +export async function parseSong(source: Source): Promise { + return parseSongFromEntry(await resolveSource(source)); } diff --git a/src/parsePack.ts b/src/parsePack.ts new file mode 100644 index 0000000..937ea46 --- /dev/null +++ b/src/parsePack.ts @@ -0,0 +1,195 @@ +import { supportedExtensions } from "./parsers/index.js"; +import { parseSongFromEntry } from "./parseSong.js"; +import { Pack, Simfile } from "./types.js"; +import { reportError } from "./util.js"; +import { AnyEntry, DirLike, entriesOf, isDir } from "./vfs/index.js"; + +export type PackWithSongs = Pack & { simfiles: Simfile[] }; + +/** + * @param dir a directory to inspect + * @returns true if the directory directly contains a simfile + */ +async function containsSimfile(dir: DirLike): Promise { + for await (const entry of dir.entries()) { + if ( + !isDir(entry) && + supportedExtensions.some((ext) => entry.name.endsWith(ext)) + ) { + return true; + } + } + return false; +} + +/** + * @param dir a directory to inspect + * @returns the directory's immediate subfolders + */ +async function subdirectories(dir: DirLike): Promise { + return (await entriesOf(dir)).filter(isDir); +} + +/** + * @param dir a directory to inspect + * @returns true if any of the directory's subfolders is a song folder + */ +async function looksLikePack(dir: DirLike): Promise { + for (const subdir of await subdirectories(dir)) { + if (await containsSimfile(subdir)) { + return true; + } + } + return false; +} + +/** how many nested wrapper folders to look through before giving up */ +const maxPackDepth = 4; + +type PackSearch = + | { type: "found"; dir: DirLike } + | { type: "multiple"; packs: DirLike[] } + | { type: "none" }; + +/** + * Finds the folder that actually holds the song folders. Archives and download + * folders commonly wrap a pack in one or more extra folders, so descend + * through them until we reach a folder whose children look like songs. + * @param dir the folder to search + * @param depth how many levels have been descended so far + * @returns the pack folder, or why one couldn't be settled on + */ +async function findPackRoot(dir: DirLike, depth = 0): Promise { + if (await looksLikePack(dir)) { + return { type: "found", dir }; + } + + // nothing here is a song, so look for the pack among the subfolders. Doing + // it by what they contain rather than by counting them means junk folders + // sitting next to the pack don't make it ambiguous. + const subdirs = await subdirectories(dir); + const packs: DirLike[] = []; + for (const subdir of subdirs) { + if (await looksLikePack(subdir)) { + packs.push(subdir); + } + } + if (packs.length === 1) { + return { type: "found", dir: packs[0] }; + } + if (packs.length > 1) { + return { type: "multiple", packs }; + } + + if (subdirs.length === 1 && depth < maxPackDepth) { + return findPackRoot(subdirs[0], depth + 1); + } + return { type: "none" }; +} + +/** how many pack names to name individually before summarizing the rest */ +const maxNamesInError = 5; + +/** + * @param packs the packs that were found + * @returns an error explaining that only one pack can be parsed at a time + */ +function multiplePacksError(packs: DirLike[]): Error { + // sorted so the message doesn't depend on the order the source happens to + // list its entries in + const names = packs.map((pack) => `'${pack.name}'`).sort(); + const listed = names.slice(0, maxNamesInError).join(", "); + const rest = names.length - maxNamesInError; + return new Error( + `expected a single pack, but found ${names.length}: ` + + (rest > 0 ? `${listed}, and ${rest} more` : listed) + + ". Use parseAllPacks to parse a folder of packs.", + ); +} + +/** + * @param dir the folder a pack was found in + * @param name an explicit name to use instead of guessing from the folder + * @returns pack metadata + */ +export function packFromDir(dir: DirLike, name?: string): Pack { + return { + // an explicitly provided name is used as given, rather than being run + // through the guesswork we apply to folder names + name: name ?? dir.name.replace(/-/g, " "), + dir: dir.name, + path: dir.path, + songCount: 0, + }; +} + +/** + * Parses every song folder inside a directory into a pack, without looking for + * the pack first. + * @param dir the pack's folder + * @param pack metadata for the pack being built, mutated with the song count + * @returns parsed pack + */ +export async function parsePackDir( + dir: DirLike, + pack: Pack, +): Promise { + const songFolders = await subdirectories(dir); + + const simfiles: Simfile[] = []; + for (const songFolder of songFolders) { + try { + const songData = await parseSongFromEntry(songFolder); + if (songData) { + simfiles.push({ + ...songData, + pack, + }); + } + } catch (e) { + reportError(`parseStepchart failed for '${songFolder.name}'`, e); + } + } + + pack.songCount = simfiles.length; + + return { + ...pack, + simfiles, + }; +} + +/** + * Parse a pack from an already resolved entry, locating the pack folder first. + * @param entry the folder or opened archive to search + * @param name optional pack name, overriding the guess made from the folder + * @throws {Error} if more than one pack is found, or no songs at all + * @returns parsed pack + */ +export async function parsePackFromEntry( + entry: AnyEntry, + name?: string, +): Promise { + if (!isDir(entry)) { + throw new Error( + "expected a folder or zip archive holding a pack, but got a single file", + ); + } + + const search = await findPackRoot(entry); + if (search.type === "multiple") { + throw multiplePacksError(search.packs); + } + if (search.type === "none") { + throw new Error( + "found no songs here; expected a pack containing one folder per song", + ); + } + + // a pack whose songs sit at the root of an archive has no folder of its own + // to take a name from, so fall back to the archive's name + const dir = search.dir.name + ? search.dir + : { ...search.dir, name: entry.name }; + return parsePackDir(search.dir, packFromDir(dir, name)); +} diff --git a/src/parseSong.ts b/src/parseSong.ts index 8fb205a..cd42dd7 100644 --- a/src/parseSong.ts +++ b/src/parseSong.ts @@ -1,66 +1,85 @@ -import * as fs from "node:fs"; -import * as path from "node:path"; -import { Simfile } from "./types.js"; import { parsers, supportedExtensions, sortFileCandidatesByPriority, } from "./parsers/index.js"; -import type { ParsedImages, RawSimfile } from "./parsers/types.js"; +import { ParsedImages, RawSimfile } from "./parsers/types.js"; +import { ImageRef, Simfile } from "./types.js"; +import { extname } from "./util.js"; +import { AnyEntry, DirLike, FileLike, isDir } from "./vfs/index.js"; /** - * Find a simfile in a given directory - * @param songDir directory path - * @returns filename of the found simfile + * Find the best simfile in a given directory + * @param songDir directory to search + * @returns the most preferred simfile found, or null */ -function getSongFile(songDir: string) { - const files = fs.readdirSync(songDir); - const candidates = files - .filter((f) => supportedExtensions.some((ext) => f.endsWith(ext))) - .sort(sortFileCandidatesByPriority); - if (candidates.length) { - return candidates[0]; +async function identifySongFile(songDir: DirLike): Promise { + const candidates: FileLike[] = []; + for await (const entry of songDir.entries()) { + if ( + !isDir(entry) && + supportedExtensions.some((ext) => entry.name.endsWith(ext)) + ) { + candidates.push(entry); + } } - return null; + if (!candidates.length) { + return null; + } + candidates.sort((a, b) => sortFileCandidatesByPriority(a.name, b.name)); + return candidates[0]; } const imageExts = new Set([".png", ".jpg"]); + /** * Get all image files in a given directory - * @param songDir directory - * @returns contents filtered to supported image extentions + * @param songDir directory to search + * @yields {FileLike} each file with a supported image extension */ -function getImages(songDir: string): string[] { - const files = fs.readdirSync(songDir); - return files.filter((f) => imageExts.has(path.extname(f))); +async function* getImages(songDir: DirLike) { + for await (const entry of songDir.entries()) { + if (isDir(entry)) { + continue; + } + const ext = extname(entry.name); + if (ext && imageExts.has(ext)) { + yield entry; + } + } } /** - * Make some best guesses about which images should be used for which fields - * @param songDir path to a song directory + * Make some best guesses about which images should be used for which fields. + * The images themselves are never read here, only located. + * @param songDir the song's directory * @param tagged image metadata found in simfile * @returns final image metadata */ -function guessImages(songDir: string, tagged: ParsedImages) { - let jacket = tagged.jacket; - let bg = tagged.bg; - let banner = tagged.banner; - const leftovers: string[] = []; - for (const image of getImages(songDir)) { - const ext = path.extname(image); +async function guessImages( + songDir: DirLike, + tagged: ParsedImages, +): Promise> { + let jacket = tagged.jacket ? await songDir.getFile(tagged.jacket) : null; + let bg = tagged.bg ? await songDir.getFile(tagged.bg) : null; + let banner = tagged.banner ? await songDir.getFile(tagged.banner) : null; + const leftovers: FileLike[] = []; + for await (const image of getImages(songDir)) { + const imageName = image.name; + const ext = extname(imageName) || ""; if ( - (!jacket && image.endsWith("-jacket" + ext)) || - image.startsWith("jacket.") + (!tagged.jacket && imageName.endsWith("-jacket" + ext)) || + imageName.startsWith("jacket.") ) { jacket = image; } else if ( - (!bg && image.endsWith("-bg" + ext)) || - image.startsWith("bg.") + (!tagged.bg && imageName.endsWith("-bg" + ext)) || + imageName.startsWith("bg.") ) { bg = image; } else if ( - (!banner && image.endsWith("-bn" + ext)) || - image.startsWith("bn.") + (!tagged.bg && imageName.endsWith("-bn" + ext)) || + imageName.startsWith("bn.") ) { banner = image; } else { @@ -79,12 +98,6 @@ function guessImages(songDir: string, tagged: ParsedImages) { return { jacket, bg, banner }; } -// function toSafeName(name: string): string { -// name = name.replace(".png", ""); -// name = name.replace(/\s/g, "-").replace(/[^\w]/g, "_"); -// return `${name}.png`; -// } - /** * get individual bpms of each chart * @param sm simfile @@ -96,17 +109,22 @@ function getBpms(sm: Pick): number[] { } /** - * Parse a single simfile. Automatically determines which parser to use depending on chart definition type. - * @param songDirPath path to song folder (contains a chart definition file [dwi/sm], images, etc) - * @returns a simfile object without mix info or null if no sm/ssc file was found + * Parse a single song from an already resolved entry. Automatically determines + * which parser to use depending on chart definition type. + * @param songDirOrFile a song folder, or a single chart file + * @returns a simfile object without mix info, or null if no chart was found */ -export function parseSong(songDirPath: string): Simfile | null { - const songFile = getSongFile(songDirPath); - if (!songFile) { - return null; - } - const stepchartPath = path.join(songDirPath, songFile); - const extension = path.extname(stepchartPath); +export async function parseSongFromEntry( + songDirOrFile: AnyEntry, +): Promise { + const songDir = isDir(songDirOrFile) ? songDirOrFile : null; + const songFile = songDir + ? await identifySongFile(songDir) + : (songDirOrFile as FileLike); + if (!songFile) return null; + + const extension = extname(songFile.name); + if (!extension) return null; const parser = parsers[extension]; @@ -114,10 +132,10 @@ export function parseSong(songDirPath: string): Simfile | null { throw new Error(`No parser registered for extension: ${extension}`); } - const fileContents = fs.readFileSync(stepchartPath); + const file = await songFile.file(); const { images, ...rawStepchart } = parser( - fileContents.toString(), - songDirPath, + await file.text(), + songDirOrFile.path ?? songDirOrFile.name, ); if (!Object.keys(rawStepchart.charts).length) { @@ -140,8 +158,11 @@ export function parseSong(songDirPath: string): Simfile | null { title: { titleName: rawStepchart.title, translitTitleName: rawStepchart.titletranslit ?? null, - titleDir: songDirPath, - ...guessImages(songDirPath, images), + titleDir: songDirOrFile.name, + titlePath: songDirOrFile.path, + ...(songDir + ? await guessImages(songDir, images) + : { banner: null, bg: null, jacket: null }), }, subtitle: { subtitleName: rawStepchart.subtitle ?? "", diff --git a/src/types.ts b/src/types.ts index 856c390..6dc62d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -89,19 +89,39 @@ export interface Simfile { stats: Stats; } +/** + * A reference to an image belonging to a song. The contents are read on + * demand, so an image found while parsing is only loaded off disk or + * decompressed out of an archive if something actually asks for it. + */ +export interface ImageRef { + /** the image's own filename, e.g. `DDRMAX2-bn.png` */ + name: string; + /** where the image lives on disk, or null if it came out of an archive */ + path: string | null; + /** reads the image's contents */ + file(): Promise; +} + export interface Pack { name: string; + /** the name of the folder the pack lives in */ dir: string; + /** the pack folder's path on disk, or null if it came out of an archive */ + path: string | null; songCount: number; } export interface Title { titleName: string; translitTitleName: string | null; + /** the name of the folder the song lives in */ titleDir: string; - banner: string | null; - bg: string | null; - jacket: string | null; + /** the song folder's path on disk, or null if it came out of an archive */ + titlePath: string | null; + banner: ImageRef | null; + bg: ImageRef | null; + jacket: ImageRef | null; } export interface Subtitle { diff --git a/src/util.ts b/src/util.ts index 637131c..e844de5 100644 --- a/src/util.ts +++ b/src/util.ts @@ -98,6 +98,19 @@ export function renameBackground( } } +/** + * returns extension name from a filename + * @param name filename + * @returns extension, with leading period + */ +export function extname(name: string) { + const match = name.match(/.+(\.[^.]+)$/); + if (match) { + return match[1]; + } + return null; +} + let errorTolerance: "bail" | "warn" | "ignore" = "warn"; /** diff --git a/src/vfs/archive.ts b/src/vfs/archive.ts new file mode 100644 index 0000000..43471a6 --- /dev/null +++ b/src/vfs/archive.ts @@ -0,0 +1,176 @@ +/** + * Presents a zip archive as a virtual filesystem directory, reading entries + * out of it lazily. + */ + +import { AnyEntry, DirLike, FileLike, lenientGet, splitPath } from "./index.js"; +import { readCentralDirectory, readEntry, ZipEntry } from "./zip.js"; + +/** folders some archivers add alongside the real contents */ +const ignoredNames = new Set(["__MACOSX", ".DS_Store", "Thumbs.db"]); + +interface ZipNode { + name: string; + dirs: Map; + files: Map; + parent: ZipNode | null; +} + +/** + * @param name the node's own name + * @param parent the containing node, if any + * @returns an empty tree node + */ +function makeNode(name: string, parent: ZipNode | null): ZipNode { + return { name, dirs: new Map(), files: new Map(), parent }; +} + +/** + * Rebuilds the archive's folder hierarchy from its flat list of entries. + * Intermediate folders are created as needed, since archives are not required + * to include explicit entries for them. + * @param entries every entry in the archive + * @param rootName a name to give the root of the tree + * @returns the root node of the tree + */ +function buildTree(entries: ZipEntry[], rootName: string): ZipNode { + const root = makeNode(rootName, null); + for (const entry of entries) { + const segments = splitPath(entry.name); + if (!segments.length || segments.some((s) => ignoredNames.has(s))) { + continue; + } + const filename = entry.isDirectory ? null : segments.pop(); + let node = root; + for (const segment of segments) { + let child = node.dirs.get(segment); + if (!child) { + child = makeNode(segment, node); + node.dirs.set(segment, child); + } + node = child; + } + if (filename && !ignoredNames.has(filename)) { + node.files.set(filename, entry); + } + } + return root; +} + +/** + * @param archive the source archive + * @param node the tree node to wrap + * @returns the node as a virtual filesystem directory + */ +function dirFromZipNode(archive: Blob, node: ZipNode): DirLike { + return { + type: "directory", + name: node.name, + // nothing inside an archive has a location on disk + path: null, + async *entries(): AsyncIterable { + for (const child of node.dirs.values()) { + yield dirFromZipNode(archive, child); + } + for (const [name, entry] of node.files) { + yield fileFromZipEntry(archive, name, entry); + } + }, + getFile(path) { + const segments = splitPath(path); + const filename = segments.pop(); + if (!filename) { + return Promise.resolve(null); + } + let dir: ZipNode | null | undefined = node; + for (const segment of segments) { + dir = segment === ".." ? dir.parent : lenientGet(dir.dirs, segment); + if (!dir) { + return Promise.resolve(null); + } + } + const entry = lenientGet(dir.files, filename); + if (!entry) { + return Promise.resolve(null); + } + // report the name the archive actually holds rather than the one that + // was asked for, which may differ in case + const actual = splitPath(entry.name).pop() ?? filename; + return Promise.resolve(fileFromZipEntry(archive, actual, entry)); + }, + }; +} + +/** + * Decompressed entries, keyed by the entry they came from. A single image + * routinely gets picked for more than one role in a song, and looking one up + * twice hands back two separate wrappers around the same entry, so without + * this it would be decompressed once per use. + */ +const readEntries = new WeakMap>(); + +/** + * @param archive the source archive + * @param name the entry's own filename + * @param entry the entry to wrap + * @returns the entry as a virtual filesystem file, read lazily + */ +function fileFromZipEntry( + archive: Blob, + name: string, + entry: ZipEntry, +): FileLike { + return { + type: "file", + name, + path: null, + file() { + let pending = readEntries.get(entry); + if (!pending) { + pending = readEntry(archive, entry).then( + (contents) => new File([contents], name), + ); + readEntries.set(entry, pending); + } + return pending; + }, + }; +} + +/** + * Opens a zip archive as a virtual filesystem directory. Only the archive's + * index is read here; entries are decompressed individually, on demand. + * @param archive the zip file + * @param name a name for the archive's root directory + * @returns the root of the archive + */ +export async function openZip(archive: Blob, name = ""): Promise { + const entries = await readCentralDirectory(archive); + return dirFromZipNode(archive, buildTree(entries, name)); +} + +/** + * @param blob a file that may or may not be a zip archive + * @returns true if the file starts with the zip magic number + */ +export async function isZip(blob: Blob): Promise { + if (blob.size < 4) { + return false; + } + const magic = new Uint8Array(await blob.slice(0, 4).arrayBuffer()); + // "PK\x03\x04", the local file header signature every zip starts with + return ( + magic[0] === 0x50 && + magic[1] === 0x4b && + magic[2] === 0x03 && + magic[3] === 0x04 + ); +} + +/** + * @param filename name of a zip file + * @returns the name with any `.zip` extension removed + */ +export function stripZipExtension(filename: string) { + return filename.replace(/\.zip$/i, ""); +} diff --git a/src/vfs/dom.ts b/src/vfs/dom.ts new file mode 100644 index 0000000..b5f708a --- /dev/null +++ b/src/vfs/dom.ts @@ -0,0 +1,185 @@ +/** + * Adapters for the two filesystem APIs a browser might hand us: the modern + * File System Access handles, and the older drag & drop entries. + */ + +import { AnyEntry, DirLike, FileLike, splitPath } from "./index.js"; + +// --- File System Access API (handles) --------------------------------------- + +/** + * @param handle a file handle + * @returns the handle as a virtual filesystem file + */ +function fileFromHandle(handle: FileSystemFileHandle): FileLike { + return { + type: "file", + name: handle.name, + // the browser never exposes a real path + path: null, + file: () => handle.getFile(), + }; +} + +/** + * @param handle a directory handle + * @returns the handle as a virtual filesystem directory + */ +function dirFromHandle(handle: FileSystemDirectoryHandle): DirLike { + return { + type: "directory", + name: handle.name, + path: null, + async *entries() { + for await (const child of handle.values()) { + yield fromHandle(child); + } + }, + async getFile(path) { + const segments = splitPath(path); + const filename = segments.pop(); + if (!filename) { + return null; + } + try { + let dir = handle; + for (const segment of segments) { + if (segment === "..") { + // this api gives no way to walk up out of the granted directory + return null; + } + dir = await dir.getDirectoryHandle(segment); + } + return fileFromHandle(await dir.getFileHandle(filename)); + } catch { + return null; + } + }, + }; +} + +/** + * @param handle any file system handle + * @returns the handle as a virtual filesystem entry + */ +export function fromHandle(handle: FileSystemHandle): AnyEntry { + return handle.kind === "directory" + ? dirFromHandle(handle as FileSystemDirectoryHandle) + : fileFromHandle(handle as FileSystemFileHandle); +} + +// --- legacy drag & drop entries --------------------------------------------- + +/** + * @param entry a file entry + * @returns the entry as a virtual filesystem file + */ +function fileFromEntry(entry: FileSystemFileEntry): FileLike { + return { + type: "file", + name: entry.name, + path: null, + file: () => new Promise((resolve, reject) => entry.file(resolve, reject)), + }; +} + +/** + * `readEntries` only returns a limited number of children per call, so it has + * to be called until it comes back empty to see a whole directory. + * @param dir a directory entry + * @returns every child of the directory + */ +function readAllEntries(dir: FileSystemDirectoryEntry) { + const reader = dir.createReader(); + const all: FileSystemEntry[] = []; + return new Promise((resolve, reject) => { + const readBatch = () => + reader.readEntries((batch) => { + if (!batch.length) { + resolve(all); + return; + } + all.push(...batch); + readBatch(); + }, reject); + readBatch(); + }); +} + +/** + * @param entry a directory entry + * @returns the entry as a virtual filesystem directory + */ +function dirFromEntry(entry: FileSystemDirectoryEntry): DirLike { + return { + type: "directory", + name: entry.name, + path: null, + async *entries() { + for (const child of await readAllEntries(entry)) { + yield fromEntry(child); + } + }, + async getFile(path) { + const segments = splitPath(path); + try { + let dir = entry; + while (segments[0] === "..") { + segments.shift(); + dir = await new Promise((resolve, reject) => + dir.getParent(resolve as never, reject), + ); + } + if (!segments.length) { + return null; + } + const found = await new Promise((resolve, reject) => + dir.getFile(segments.join("/"), {}, resolve, reject), + ); + return found.isFile + ? fileFromEntry(found as FileSystemFileEntry) + : null; + } catch { + return null; + } + }, + }; +} + +/** + * @param entry any file system entry + * @returns the entry as a virtual filesystem entry + */ +export function fromEntry(entry: FileSystemEntry): AnyEntry { + return entry.isDirectory + ? dirFromEntry(entry as FileSystemDirectoryEntry) + : fileFromEntry(entry as FileSystemFileEntry); +} + +// --- plain files ------------------------------------------------------------ + +/** + * @param file a file + * @returns the file as a virtual filesystem file + */ +export function fromFile(file: File): FileLike { + return { + type: "file", + name: file.name, + path: null, + file: () => Promise.resolve(file), + }; +} + +/** + * @param source anything a browser might hand us for a dropped item + * @returns the item as a virtual filesystem entry + */ +export function fromDom( + source: FileSystemHandle | FileSystemEntry | File, +): AnyEntry { + if (source instanceof File) { + return fromFile(source); + } + return "kind" in source ? fromHandle(source) : fromEntry(source); +} diff --git a/src/vfs/index.ts b/src/vfs/index.ts new file mode 100644 index 0000000..2b9d0e4 --- /dev/null +++ b/src/vfs/index.ts @@ -0,0 +1,96 @@ +/** + * A tiny virtual filesystem the parsers target, so they don't have to care + * whether a song came from a folder on disk, a zip archive, or one of the + * browser's two filesystem APIs. Each source gets an adapter in this folder. + */ + +import { ImageRef } from "../types.js"; + +export interface FileLike extends ImageRef { + type: "file"; +} + +export interface DirLike { + type: "directory"; + name: string; + /** where the directory lives on disk, or null if it came out of an archive */ + path: string | null; + /** iterates the directory's immediate children */ + entries(): AsyncIterable; + /** + * resolves a path relative to this directory, which may include `..` + * segments, to a file. Resolves to null if it can't be found. + */ + getFile(path: string): Promise; +} + +export type AnyEntry = FileLike | DirLike; + +/** + * @param entry any virtual filesystem entry + * @returns true if the entry is a directory + */ +export function isDir(entry: AnyEntry): entry is DirLike { + return entry.type === "directory"; +} + +/** + * Blobs and Files carry a `type` of their own — their mime type — so telling + * one from a virtual filesystem entry takes more than a property check. + * @param source anything that might already be an entry + * @returns true if it is one + */ +export function isEntry(source: unknown): source is AnyEntry { + return ( + typeof source === "object" && + source !== null && + !(source instanceof Blob) && + "type" in source && + (source.type === "file" || source.type === "directory") + ); +} + +/** + * Splits a simfile-relative path into segments, tolerating the backslashes + * that occasionally show up in tags authored on Windows. + * @param path a relative path + * @returns the meaningful path segments + */ +export function splitPath(path: string): string[] { + return path.split(/[/\\]/).filter((segment) => segment && segment !== "."); +} + +/** + * Looks up a key in a map, falling back to a case insensitive match. Packs are + * routinely authored on case insensitive filesystems, so a simfile's tags may + * disagree with the filesystem on the casing of a filename. + * @param map the map to search + * @param key the key to look for + * @returns the matching value, or undefined + */ +export function lenientGet(map: Map, key: string): T | undefined { + const exact = map.get(key); + if (exact !== undefined) { + return exact; + } + const lowered = key.toLowerCase(); + for (const [candidate, value] of map) { + if (candidate.toLowerCase() === lowered) { + return value; + } + } + return undefined; +} + +/** + * Collects a directory's children, which adapters expose as an async iterable. + * @param dir a directory + * @returns every immediate child of the directory + */ +export async function entriesOf(dir: DirLike): Promise { + const all: AnyEntry[] = []; + for await (const entry of dir.entries()) { + all.push(entry); + } + return all; +} diff --git a/src/vfs/node.ts b/src/vfs/node.ts new file mode 100644 index 0000000..7fd81ff --- /dev/null +++ b/src/vfs/node.ts @@ -0,0 +1,139 @@ +/** + * Adapter for folders and archives on disk. This is the only part of the + * parser that touches `node:fs`, so nothing here may be imported from the + * browser entry point. + */ + +import * as fs from "node:fs/promises"; +import { openAsBlob } from "node:fs"; +import * as path from "node:path"; +import { AnyEntry, DirLike, FileLike, isEntry, splitPath } from "./index.js"; +import { isZip, openZip, stripZipExtension } from "./archive.js"; + +/** anything the node entry points will parse from */ +export type Source = string | Blob | AnyEntry; + +/** + * @param filePath path to a file + * @returns the file as a virtual filesystem file, read on demand + */ +function fileFromPath(filePath: string): FileLike { + const name = path.basename(filePath); + return { + type: "file", + name, + path: filePath, + async file() { + return new File([await fs.readFile(filePath)], name); + }, + }; +} + +/** + * Finds a file whose name differs from the one asked for only by case. Packs + * are routinely authored on case insensitive filesystems, so a simfile's tags + * may disagree with the disk about the casing of an image filename. + * @param dirPath the directory to search + * @param filename the filename to match + * @returns the real filename on disk, or null + */ +async function findLeniently( + dirPath: string, + filename: string, +): Promise { + const lowered = filename.toLowerCase(); + try { + for (const candidate of await fs.readdir(dirPath)) { + if (candidate.toLowerCase() === lowered) { + return candidate; + } + } + } catch { + // the directory itself is missing, which getFile reports as a miss + } + return null; +} + +/** + * @param dirPath path to a directory + * @returns the directory as a virtual filesystem directory + */ +export function dirFromPath(dirPath: string): DirLike { + return { + type: "directory", + name: path.basename(dirPath), + path: dirPath, + async *entries(): AsyncIterable { + for (const name of await fs.readdir(dirPath)) { + const full = path.join(dirPath, name); + // stat rather than readdir's dirent so symlinked folders are followed, + // which is how this behaved before the virtual filesystem existed + const stats = await fs.stat(full).catch(() => null); + if (!stats) { + continue; + } + yield stats.isDirectory() ? dirFromPath(full) : fileFromPath(full); + } + }, + async getFile(relative) { + const segments = splitPath(relative); + const filename = segments.pop(); + if (!filename) { + return null; + } + const parent = path.resolve(dirPath, ...segments); + const target = path.join(parent, filename); + const stats = await fs.stat(target).catch(() => null); + if (stats?.isFile()) { + return fileFromPath(target); + } + if (stats) { + // it exists but is a directory + return null; + } + const lenient = await findLeniently(parent, filename); + return lenient ? fileFromPath(path.join(parent, lenient)) : null; + }, + }; +} + +/** + * Turns whatever the caller passed into something the parsers can read: a + * folder on disk, a zip archive by path or in memory, or a single file. + * @param source a path, a `Blob`/`File`, or an already resolved entry + * @returns the source as a virtual filesystem entry + */ +export async function resolveSource(source: Source): Promise { + if (isEntry(source)) { + return source; + } + + if (typeof source !== "string") { + if (await isZip(source)) { + const name = source instanceof File ? stripZipExtension(source.name) : ""; + return openZip(source, name); + } + if (source instanceof File) { + return { + type: "file", + name: source.name, + path: null, + file: () => Promise.resolve(source), + }; + } + throw new Error( + "expected a zip archive, but the data provided was not one", + ); + } + + const stats = await fs.stat(source); + if (stats.isDirectory()) { + return dirFromPath(source); + } + + const blob = await openAsBlob(source); + if (await isZip(blob)) { + return openZip(blob, stripZipExtension(path.basename(source))); + } + return fileFromPath(source); +} diff --git a/src/browser/zip.ts b/src/vfs/zip.ts similarity index 100% rename from src/browser/zip.ts rename to src/vfs/zip.ts