From e05122229802d87f068a81e785fd75117313d857 Mon Sep 17 00:00:00 2001 From: 12problems Date: Mon, 17 Aug 2026 22:03:39 -0400 Subject: [PATCH] Hash mods' flattened folder content instead of a rebuilt zip The launcher no longer deploys mods as zips - Steamodded mounting a .zip via NFS.mount() didn't behave correctly for every mod in practice, so it now deploys real extracted folders instead (see the companion launcher-repo PR). The "approved hash" this server computes has to match what the launcher actually verifies against, so it switches too: hash a mod's flattened content directly instead of assembling and hashing a deterministic zip of it. - New mod-folder-hash.ts: computeModFolderHash() is a byte-for-byte Node port of the launcher's ModFileHash::hashDirectory() - sorted relative POSIX paths + file bytes, fed into one running SHA-256. Verified against the launcher side with a real cross-language parity check (not just each side's own unit tests): a standalone build of the actual modfilehash.cpp and a Node script calling this file produced identical digests for the same sample mod tree, both before and after renaming the tree's own root folder (which deliberately doesn't factor into the hash on either side, unlike the old zip-based scheme where the two had to agree on folder naming for the resulting hash to match at all). - mods-sync.service.ts: computePreparedZipHash() -> renamed computeModFolderHashForRelease(), now calls computeModFolderHash() on the flattened extracted directory directly instead of shelling out to modzip and hashing the assembled archive's bytes. - Deleted native/modzip/ and its Dockerfile build step entirely - no native C binary/libzip to compile or ship anymore. - mod-archive-flatten.ts/backfill-mod-hashes.ts doc comments updated to match; relocateModRoot() itself is unchanged (still needed to produce the same flattened layout the launcher's own extraction step would). - New mod-folder-hash.test.ts: hash-vs-hand-computed-SHA-256, root-name independence, enumeration-order independence, content/rename sensitivity. All passing. No DB schema change - modRegistry.latestSha256/modRegistryVersions.sha256 stay hex-sha256 columns, just fed different input bytes now. Every hash stored under the old algorithm is wrong under this one - `pnpm backfill-mod-hashes` needs to run once after this deploys (see backfill-mod-hashes.ts, unchanged, already idempotent). Co-Authored-By: Claude Sonnet 5 --- apps/server/Dockerfile | 12 - apps/server/native/modzip/modzip.c | 335 ------------------ .../src/features/mods/backfill-mod-hashes.ts | 28 +- .../src/features/mods/mod-archive-flatten.ts | 4 +- .../src/features/mods/mod-folder-hash.ts | 59 +++ .../src/features/mods/mods-sync.service.ts | 89 ++--- .../tests/services/mod-folder-hash.test.ts | 88 +++++ 7 files changed, 199 insertions(+), 416 deletions(-) delete mode 100644 apps/server/native/modzip/modzip.c create mode 100644 apps/server/src/features/mods/mod-folder-hash.ts create mode 100644 apps/server/src/tests/services/mod-folder-hash.test.ts diff --git a/apps/server/Dockerfile b/apps/server/Dockerfile index e11256f4..3d797e54 100644 --- a/apps/server/Dockerfile +++ b/apps/server/Dockerfile @@ -14,18 +14,6 @@ RUN pnpm install --frozen-lockfile --filter balatro-multiplayer-api-server... COPY packages/ ./packages/ COPY apps/server/ ./apps/server/ -# modzip: the deterministic mod-archive builder mods-sync.service.ts shells -# out to (see native/modzip/modzip.c) - links libzip so the hash it -# produces matches the launcher's own ZipWriter::zipDirectory() byte-for- -# byte. build-base (gcc/make) is only needed to compile it; removed -# afterward to keep the final image lean. libzip-dev is left installed -# since it's also how libzip's runtime shared library gets pulled in on -# Alpine - removing it risks taking the .so modzip needs at runtime with it. -RUN apk add --no-cache libzip-dev build-base \ - && gcc -O2 -std=c11 -D_POSIX_C_SOURCE=200809L -Wall -Wextra \ - -o /usr/local/bin/modzip apps/server/native/modzip/modzip.c -lzip \ - && apk del build-base - RUN pnpm --filter @bmp/types build RUN pnpm --filter @v-rtualized/bmp-internal build RUN pnpm --filter balatro-multiplayer-api-server build diff --git a/apps/server/native/modzip/modzip.c b/apps/server/native/modzip/modzip.c deleted file mode 100644 index 56d20d0a..00000000 --- a/apps/server/native/modzip/modzip.c +++ /dev/null @@ -1,335 +0,0 @@ -// Deterministic, cross-platform-independent mod archive builder, matching -// the launcher's own ZipWriter::zipDirectory() (see -// new-launcher/src/mods/zipwriter.cpp) byte-for-byte: same library -// (libzip), same fixed per-entry mtime, same compression method/level, -// same sorted entry order, same single-top-level-wrapper-folder layout. -// -// This server hashes what this program produces instead of hashing the raw -// GitHub release archive, because the launcher never actually deploys that -// raw archive into a user's Mods folder -- it always extracts, flattens -// (drops wrapper/README/LICENSE clutter, promotes the real -// .lua-containing folder to the top -- see -// ../../src/features/mods/mod-archive-flatten.ts, a TypeScript port of the -// launcher's relocateModRoot()), and rezips first. The rezip step is what -// RunController::currentZipMatchesServerHash() actually hashes for Ranked -// verification, so that's what this needs to match, not the original -// download. -// -// Usage: modzip -// Writes the assembled zip's raw bytes to stdout. Diagnostics go to -// stderr. Exit code 0 on success, 1 on any failure (missing/unreadable -// sourceDir, a file read failure, a libzip error). sourceDir's contents -// should already be flattened by the caller before this runs -- this -// program only does the deterministic archive-building step. -// -// Deliberately no target-file argument, unlike ZipWriter::zipDirectory(): -// the server only ever needs the resulting bytes to hash (see -// mods-sync.service.ts), never a persisted zip file, so writing straight to -// stdout avoids a temp-file round trip on the Node side. - -#define _POSIX_C_SOURCE 200809L - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -// --- Growable array of heap-owned strings: the collected list of paths -// relative to sourceDir (see walk()). --- - -typedef struct { - char **items; - size_t count; - size_t capacity; -} string_list; - -static void string_list_push(string_list *list, char *item) -{ - if (list->count == list->capacity) { - list->capacity = list->capacity ? list->capacity * 2 : 64; - list->items = realloc(list->items, list->capacity * sizeof(char *)); - if (!list->items) { - fprintf(stderr, "modzip: out of memory\n"); - exit(1); - } - } - list->items[list->count++] = item; -} - -// Plain byte-wise comparison -- equivalent to Qt's UTF-16 code-unit -// QString::operator< for every codepoint below U+10000 (i.e. every -// realistic mod filename), which is what ZipWriter::zipDirectory()'s -// std::sort(relativePaths) actually sorts by. -static int compare_strings(const void *a, const void *b) -{ - return strcmp(*(const char *const *)a, *(const char *const *)b); -} - -// Recursive walk of dir_path (starting at source_root itself), collecting -// every regular file's path relative to source_root, forward-slash -// separated (this only ever runs on Linux, so that's already the native -// separator). Mirrors QDirIterator(path, QDir::Files | QDir::NoDotAndDotDot, -// QDirIterator::Subdirectories): only regular files are collected -- no -// directory entries, since an archive's directories are always implicit -// from its file entries' paths, same as the Qt version. Symlinks are -// deliberately not followed (lstat, not stat): a mod archive legitimately -// containing one is vanishingly unlikely, and following one would open the -// door to escaping source_root entirely. -static void walk(const char *source_root, const char *dir_path, string_list *out) -{ - DIR *dir = opendir(dir_path); - if (!dir) { - fprintf(stderr, "modzip: couldn't read directory %s: %s\n", dir_path, strerror(errno)); - exit(1); - } - - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { - continue; - } - - size_t full_len = strlen(dir_path) + 1 + strlen(entry->d_name) + 1; - char *full_path = malloc(full_len); - snprintf(full_path, full_len, "%s/%s", dir_path, entry->d_name); - - struct stat st; - if (lstat(full_path, &st) != 0) { - fprintf(stderr, "modzip: couldn't stat %s: %s\n", full_path, strerror(errno)); - exit(1); - } - - if (S_ISDIR(st.st_mode)) { - walk(source_root, full_path, out); - free(full_path); - } else if (S_ISREG(st.st_mode)) { - // full_path is "/" -- source_root - // never ends in '/' (main() strips it), so the relative part - // starts right after source_root's length plus the separator. - const char *relative = full_path + strlen(source_root) + 1; - string_list_push(out, strdup(relative)); - free(full_path); - } else { - free(full_path); - } - } - - closedir(dir); -} - -// Strips any trailing slash and returns the final path component -- -// mirrors QFileInfo(sourceDir).fileName(), which becomes the archive's -// single top-level wrapper folder name. See zipwriter.h's comment on why -// the *name* itself doesn't matter to Steamodded's loader, only that -// exactly one such folder exists -- it still has to match exactly here for -// the resulting hash to match what the launcher produces, though. -static const char *basename_of(const char *path) -{ - size_t len = strlen(path); - for (size_t i = len; i > 0; i--) { - if (path[i - 1] == '/') { - return path + i; - } - } - return path; -} - -static void fail_zip_error(zip_error_t *err, const char *what) -{ - fprintf(stderr, "modzip: %s: %s\n", what, zip_error_strerror(err)); - zip_error_fini(err); - exit(1); -} - -int main(int argc, char **argv) -{ - if (argc != 2) { - fprintf(stderr, "usage: modzip \n"); - return 1; - } - - // libzip's zip_file_set_mtime() packs the given time_t into the zip - // entry's classic DOS date/time fields via the C library's *local* - // time conversion, not UTC - confirmed empirically (same input bytes, - // same fixed_mtime constant below, three different TZ env values - // below produced three different embedded timestamps and therefore - // three different archive hashes: 2000-01-01T00:00:00 read back as - // 1999-12-31T19:00:00 under America/New_York, 2000-01-01T09:00:00 - // under Asia/Tokyo). A fixed time_t constant alone does not make the - // output timezone-independent - forcing the process's own TZ to UTC - // here does. Set before any zip_* call, and must happen in-process - // (an inherited TZ env var from the caller can't be relied on). - setenv("TZ", "UTC", 1); - tzset(); - - // Strip any trailing slash up front so basename_of() and the recursive - // walk's prefix-stripping math both see a consistent, unambiguous - // source_root. - char source_root[PATH_MAX]; - strncpy(source_root, argv[1], sizeof(source_root) - 1); - source_root[sizeof(source_root) - 1] = '\0'; - size_t root_len = strlen(source_root); - while (root_len > 1 && source_root[root_len - 1] == '/') { - source_root[--root_len] = '\0'; - } - - struct stat root_stat; - if (stat(source_root, &root_stat) != 0 || !S_ISDIR(root_stat.st_mode)) { - fprintf(stderr, "modzip: %s is not a directory\n", source_root); - return 1; - } - - string_list relative_paths = {0}; - walk(source_root, source_root, &relative_paths); - qsort(relative_paths.items, relative_paths.count, sizeof(char *), compare_strings); - - const char *wrapper_name = basename_of(source_root); - - zip_error_t zerr; - zip_error_init(&zerr); - - zip_source_t *archive_source = zip_source_buffer_create(NULL, 0, 0, &zerr); - if (!archive_source) { - fail_zip_error(&zerr, "couldn't create archive buffer"); - } - // zip_open_from_source() below effectively consumes one reference; - // this extra one keeps the finished archive's bytes readable - // afterward -- mirrors zipwriter.cpp's own zip_source_keep() call. - zip_source_keep(archive_source); - - zip_t *archive = zip_open_from_source(archive_source, ZIP_TRUNCATE, &zerr); - if (!archive) { - fail_zip_error(&zerr, "couldn't open archive"); - } - zip_error_fini(&zerr); - - // Fixed epoch (2000-01-01T00:00:00Z) for every entry's mtime, and a - // pinned explicit compression level -- exactly matching zipwriter.cpp's - // kFixedEntryMtime/kCompressionLevel. A real filesystem mtime (varies - // by extraction time/host clock) or an unpinned "default" compression - // level would make an otherwise byte-identical mod hash differently - // depending on when/where it was processed, defeating the entire point - // of this rewrite. - const time_t fixed_mtime = 946684800; - const zip_uint32_t compression_level = 9; - - for (size_t i = 0; i < relative_paths.count; i++) { - const char *relative = relative_paths.items[i]; - - size_t full_len = strlen(source_root) + 1 + strlen(relative) + 1; - char *full_path = malloc(full_len); - snprintf(full_path, full_len, "%s/%s", source_root, relative); - - FILE *f = fopen(full_path, "rb"); - if (!f) { - fprintf(stderr, "modzip: couldn't open %s: %s\n", full_path, strerror(errno)); - zip_discard(archive); - return 1; - } - fseek(f, 0, SEEK_END); - long size = ftell(f); - fseek(f, 0, SEEK_SET); - if (size < 0) { - fprintf(stderr, "modzip: couldn't determine size of %s\n", full_path); - fclose(f); - zip_discard(archive); - return 1; - } - - // libzip owns this heap buffer from here (freep=1 below) -- it has - // to stay valid until zip_close() actually assembles the archive, - // well after this loop returns. - void *data = malloc(size > 0 ? (size_t)size : 1); - if (!data) { - fprintf(stderr, "modzip: out of memory reading %s\n", full_path); - fclose(f); - zip_discard(archive); - return 1; - } - size_t nread = size > 0 ? fread(data, 1, (size_t)size, f) : 0; - fclose(f); - if (nread != (size_t)size) { - fprintf(stderr, "modzip: short read on %s\n", full_path); - free(data); - zip_discard(archive); - return 1; - } - free(full_path); - - zip_source_t *entry_source = zip_source_buffer(archive, data, (zip_uint64_t)size, 1); - if (!entry_source) { - fprintf(stderr, "modzip: %s\n", zip_strerror(archive)); - free(data); - zip_discard(archive); - return 1; - } - - size_t entry_name_len = strlen(wrapper_name) + 1 + strlen(relative) + 1; - char *entry_name = malloc(entry_name_len); - snprintf(entry_name, entry_name_len, "%s/%s", wrapper_name, relative); - - zip_int64_t index = zip_file_add(archive, entry_name, entry_source, ZIP_FL_ENC_UTF_8); - free(entry_name); - if (index < 0) { - fprintf(stderr, "modzip: %s\n", zip_strerror(archive)); - zip_source_free(entry_source); // only needed on failure -- zip_file_add() owns it on success - zip_discard(archive); - return 1; - } - - zip_file_set_mtime(archive, (zip_uint64_t)index, fixed_mtime, 0); - zip_set_file_compression(archive, (zip_uint64_t)index, ZIP_CM_DEFLATE, compression_level); - } - - if (zip_close(archive) != 0) { - fprintf(stderr, "modzip: %s\n", zip_strerror(archive)); - zip_discard(archive); - return 1; - } - // On success archive is freed by zip_close() itself; the finished - // bytes now live inside archive_source, kept alive by the extra - // zip_source_keep() reference above. - - if (zip_source_open(archive_source) < 0) { - fprintf(stderr, "modzip: couldn't reopen the assembled archive buffer\n"); - zip_source_free(archive_source); - return 1; - } - - zip_stat_t stat_buf; - zip_stat_init(&stat_buf); - zip_source_stat(archive_source, &stat_buf); - zip_int64_t total_size = (stat_buf.valid & ZIP_STAT_SIZE) ? (zip_int64_t)stat_buf.size : 0; - - char *buffer = malloc(total_size > 0 ? (size_t)total_size : 1); - zip_int64_t total_read = 0; - while (total_read < total_size) { - zip_int64_t n = zip_source_read(archive_source, buffer + total_read, - (zip_uint64_t)(total_size - total_read)); - if (n <= 0) { - break; - } - total_read += n; - } - zip_source_close(archive_source); - zip_source_free(archive_source); - - if (total_read != total_size || total_size == 0) { - fprintf(stderr, "modzip: couldn't read back the assembled archive buffer\n"); - return 1; - } - - if (fwrite(buffer, 1, (size_t)total_size, stdout) != (size_t)total_size) { - fprintf(stderr, "modzip: short write to stdout\n"); - return 1; - } - fflush(stdout); - - return 0; -} diff --git a/apps/server/src/features/mods/backfill-mod-hashes.ts b/apps/server/src/features/mods/backfill-mod-hashes.ts index 3fe69e1b..264809cf 100644 --- a/apps/server/src/features/mods/backfill-mod-hashes.ts +++ b/apps/server/src/features/mods/backfill-mod-hashes.ts @@ -1,25 +1,25 @@ /** * Recomputes every mod_registry_versions row's sha256 under the corrected - * "prepared zip" algorithm (see mods-sync.service.ts's - * computePreparedZipHash doc comment) -- every hash stored before that - * rewrite was computed over the raw GitHub download, which - * RunController::currentZipMatchesServerHash() (new-launcher) never - * actually verifies against. + * folder-content-hash algorithm (see mods-sync.service.ts's + * computeModFolderHashForRelease doc comment and mod-folder-hash.ts) -- + * every hash stored before that rewrite was computed over an archive (the + * raw GitHub download, and later a rebuilt deterministic zip -- see git + * history), never a plain directory's content, which is never what + * RunController::currentModMatchesServerHash() (new-launcher) actually + * verifies against now. * * One-time maintenance operation, not part of the regular hourly/startup - * sync -- run it explicitly, once, after deploying the prepared-zip-hash - * rewrite. Safe to re-run: recomputing an already-correct hash just - * produces the same value again. + * sync -- run it explicitly, once, after deploying the folder-hash rewrite. + * Safe to re-run: recomputing an already-correct hash just produces the + * same value again. * - * Needs the same runtime as the server itself -- the modzip binary on - * PATH (only present in the built Docker image, see Dockerfile) and - * network access to every mod's GitHub download URL -- so run it inside - * the deployed container, e.g.: + * Needs the same runtime as the server itself -- network access to every + * mod's GitHub download URL -- so run it inside the deployed container, + * e.g.: * * docker compose exec api pnpm --filter balatro-multiplayer-api-server backfill-mod-hashes * - * or locally against a real DATABASE_URL if you have modzip built and on - * PATH some other way: + * or locally against a real DATABASE_URL: * * tsx --env-file=.env src/features/mods/backfill-mod-hashes.ts */ diff --git a/apps/server/src/features/mods/mod-archive-flatten.ts b/apps/server/src/features/mods/mod-archive-flatten.ts index cbde0d6e..04263372 100644 --- a/apps/server/src/features/mods/mod-archive-flatten.ts +++ b/apps/server/src/features/mods/mod-archive-flatten.ts @@ -5,8 +5,8 @@ import path from 'node:path' // findShallowestLuaDirs() / flattenSingleRootFolder() (see // new-launcher/src/mods/modinstaller.cpp) - this has to produce the exact // same on-disk layout the launcher's own extraction step would, since -// mods-sync.service.ts feeds the result straight into modzip (see -// native/modzip/modzip.c), and the resulting hash only means anything if +// mods-sync.service.ts feeds the result straight into computeModFolderHash() +// (see mod-folder-hash.ts), and the resulting hash only means anything if // this matches what a real install actually flattens a mod archive into. // // Safety bound on how many nested wrapper folders flattenSingleRootFolder() diff --git a/apps/server/src/features/mods/mod-folder-hash.ts b/apps/server/src/features/mods/mod-folder-hash.ts new file mode 100644 index 00000000..00cd59ab --- /dev/null +++ b/apps/server/src/features/mods/mod-folder-hash.ts @@ -0,0 +1,59 @@ +import { createHash } from 'node:crypto' +import { promises as fs } from 'node:fs' +import path from 'node:path' + +// Canonical directory-content hash - a byte-for-byte Node port of the +// launcher's ModFileHash::hashDirectory() (see +// new-launcher/src/mods/modfilehash.cpp). This is what a mod's "approved +// hash" is computed over now: since mods deploy as real extracted folders +// (Steamodded mounting a .zip via NFS.mount() doesn't work correctly for +// every mod - see mods-sync.service.ts's own comment), there's no archive +// container left whose bytes would even be meaningful to hash, and hashing +// the folder's actual content directly sidesteps the entire +// "different zip tools produce different bytes for identical content" +// problem the old modzip/ZipWriter machinery existed to solve - a plain +// directory has no such ambiguity. +// +// Algorithm (must match ModFileHash::hashDirectory() exactly): +// 1. Recursively collect every regular file under root, as a path +// relative to root, forward-slash separated (already true of Node's +// own path.join on Linux, which is the only platform this runs on - +// see Dockerfile). +// 2. Sort those relative paths with a plain default string sort - +// equivalent to Qt's UTF-16 code-unit QString::operator< for every +// realistic mod filename (confirmed by modzip.c's own predecessor +// comment making the same claim about strcmp; Array.prototype.sort()'s +// default UTF-16-code-unit comparison is the same equivalence). +// 3. Feed one running sha256 hash `relativePath (utf8 bytes) + file +// contents`, per file, in sorted order. Note the root folder's own +// name never enters the hash at all (paths are relative to it) - +// unlike the old zip-based scheme, this means the launcher's and this +// server's extracted-folder naming no longer has to match for the +// hash to agree. +async function collectRelativeFilePaths(root: string, dir: string, out: string[]): Promise { + const entries = await fs.readdir(dir, { withFileTypes: true }) + for (const entry of entries) { + const fullPath = path.join(dir, entry.name) + if (entry.isDirectory()) { + await collectRelativeFilePaths(root, fullPath, out) + } else if (entry.isFile()) { + out.push(path.relative(root, fullPath).split(path.sep).join('/')) + } + // Symlinks/other entry types are skipped, same as modzip.c's old + // lstat-based walk (S_ISDIR/S_ISREG only) and Qt's own + // QDir::Files filter. + } +} + +export async function computeModFolderHash(root: string): Promise { + const relativePaths: string[] = [] + await collectRelativeFilePaths(root, root, relativePaths) + relativePaths.sort() + + const hash = createHash('sha256') + for (const relativePath of relativePaths) { + hash.update(relativePath, 'utf8') + hash.update(await fs.readFile(path.join(root, relativePath))) + } + return hash.digest('hex') +} diff --git a/apps/server/src/features/mods/mods-sync.service.ts b/apps/server/src/features/mods/mods-sync.service.ts index 149fa60b..af99ebe6 100644 --- a/apps/server/src/features/mods/mods-sync.service.ts +++ b/apps/server/src/features/mods/mods-sync.service.ts @@ -1,9 +1,6 @@ -import { execFile } from 'node:child_process' -import { createHash } from 'node:crypto' import { promises as fs } from 'node:fs' import os from 'node:os' import path from 'node:path' -import { promisify } from 'node:util' import AdmZip from 'adm-zip' import { env } from '../../env.js' import { @@ -17,10 +14,9 @@ import { } from '../../infrastructure/gateways/mods.gateway.js' import { checkCustomModVersion } from './custom-mod-version-check.service.js' import { relocateModRoot } from './mod-archive-flatten.js' +import { computeModFolderHash } from './mod-folder-hash.js' import { fetchUpstreamModIndex } from './upstream-mod-index.service.js' -const execFileAsync = promisify(execFile) - export interface ModRegistrySyncSummary { modsSynced: number hashed: number @@ -37,37 +33,33 @@ const HASH_FETCH_TIMEOUT_MS = 30_000 // this pass blocks server startup (see main.ts: the server doesn't start // accepting connections until the first sync completes). const HASH_CONCURRENCY = 8 -// Generous cap on the rebuilt zip's size -- real mods are Lua source plus -// small assets, nowhere near this; it only exists to keep a malformed or -// unexpectedly huge archive from growing an unbounded in-memory buffer. -const MAX_ZIP_SIZE_BYTES = 512 * 1024 * 1024 -// Mirrors the launcher's ModDownloadCache::sanitize() + the versionPart -// half of cachedExtractedPath() exactly (see moddownloadcache.cpp): the -// rebuilt archive's single top-level folder is named after *this*, not the -// mod's id or title, and that name is part of what modzip hashes (see -// modzip.c's basename_of()) -- it has to match byte-for-byte or the hash -// never will, even though the loader itself doesn't care what the folder's -// named. +// tmpRoot's own random suffix already guarantees uniqueness between +// concurrent hashAll() workers (including two different mods that happen to +// share a version string) -- extractedDir just needs *a* name, since unlike +// the old modzip-based scheme, the folder's own name never enters the hash +// itself (computeModFolderHash() hashes paths relative to it -- see that +// module's own comment). Kept version-derived anyway purely for +// readability if this temp dir is ever inspected mid-run. function extractedFolderName(version: string): string { const sanitized = version.replace(/[@/\\]/g, '_') return `${sanitized || '_default'}_extracted` } -// Rebuilds the archive exactly the way the launcher's ModInstaller does -// before deploying it into a user's Mods folder: downloads the raw release -// archive, extracts it, flattens/relocates its real mod-root folder (see -// mod-archive-flatten.ts, a port of relocateModRoot()), then rezips -// deterministically via modzip (a thin libzip wrapper matching the -// launcher's own ZipWriter::zipDirectory() byte-for-byte -- see -// native/modzip/modzip.c). Hashes *that* archive, not the raw download, +// Reproduces exactly what the launcher's ModInstaller deploys into a +// player's Mods folder: downloads the raw release archive, extracts it, +// flattens/relocates its real mod-root folder (see mod-archive-flatten.ts, +// a port of relocateModRoot()), then hashes that flattened folder's content +// directly (computeModFolderHash() -- a port of the launcher's own +// ModFileHash::hashDirectory()). Hashes *that*, not the raw download, // because the raw download is never what actually lands in a player's Mods -// folder, or what RunController::currentZipMatchesServerHash() verifies -// against. Best-effort like the old raw-archive hasher: a slow/dead -// download URL, an unreadable archive, or a missing modzip binary (e.g. -// local dev outside Docker, where it isn't compiled -- see Dockerfile) +// folder, or what RunController::currentModMatchesServerHash() verifies +// against -- mods now deploy as real extracted folders, not zips (NFS.mount() +// zip-mounting didn't work correctly for every mod), so there's no archive +// step left to reproduce at all past the flatten. Best-effort like the old +// raw-archive hasher: a slow/dead download URL or an unreadable archive // logs and returns null rather than failing the whole sync over one mod. -async function computePreparedZipHash( +async function computeModFolderHashForRelease( modId: string, version: string, downloadUrl: string, @@ -85,12 +77,6 @@ async function computePreparedZipHash( } const rawBytes = Buffer.from(await res.arrayBuffer()) - // tmpRoot itself is a random-suffixed unique directory (avoids - // collisions between concurrent hashAll() workers, including two - // different mods that happen to share a version string) -- - // extractedDir nested inside it is the name that actually matters, - // since modzip uses *its* basename as the archive's top-level - // wrapper folder. tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'bmp-mod-hash-')) const extractedDir = path.join(tmpRoot, extractedFolderName(version)) await fs.mkdir(extractedDir, { recursive: true }) @@ -98,12 +84,7 @@ async function computePreparedZipHash( new AdmZip(rawBytes).extractAllTo(extractedDir, true) await relocateModRoot(extractedDir) - const { stdout } = await execFileAsync('modzip', [extractedDir], { - encoding: 'buffer', - maxBuffer: MAX_ZIP_SIZE_BYTES, - }) - - return createHash('sha256').update(stdout).digest('hex') + return await computeModFolderHash(extractedDir) } catch (err) { console.error(`[mods-sync] Failed to hash ${modId}@${version}:`, err) return null @@ -149,7 +130,7 @@ async function runHashPool(candidates: HashCandidate[], skipExisting: boolean): if (existingHash) continue } - const hash = await computePreparedZipHash(modId, version, downloadUrl) + const hash = await computeModFolderHashForRelease(modId, version, downloadUrl) if (hash) { await storeComputedHash(modId, version, hash) hashed++ @@ -169,12 +150,14 @@ async function hashAll(candidates: HashCandidate[]): Promise { } // One-off maintenance operation, not part of the regular hourly/startup -// sync cycle: every hash stored before the prepared-zip-hash rewrite was -// computed over the raw GitHub download, which is never what -// RunController::currentZipMatchesServerHash() (new-launcher) actually -// verifies against (see computePreparedZipHash's doc comment) -- those -// stored values are simply wrong under the corrected algorithm, not just -// stale. This recomputes every mod_registry_versions row that has a +// sync cycle: every hash stored before the folder-hash rewrite was computed +// over an archive (first the raw GitHub download, later a rebuilt +// deterministic zip -- see git history), never a plain directory's content, +// which is never what RunController::currentModMatchesServerHash() +// (new-launcher) actually verifies against now (see +// computeModFolderHashForRelease's doc comment) -- those stored values are +// simply wrong under the corrected algorithm, not just stale. This +// recomputes every mod_registry_versions row that has a // downloadUrl, unconditionally (ignores getStoredHash's short-circuit // entirely, unlike hashAll() above) -- not just the current latest version // per mod, since a ranked mod profile can pin an exact historical version @@ -215,14 +198,14 @@ export async function recomputeAllModHashes(): Promise { // // After every mod is upserted, prunes any mod_registry row whose id wasn't // in this sync (see pruneModsMissingFrom's doc comment -- isCustom rows are -// exempt) and hashes each remaining mod's prepared archive -- every mod, not -// just ranked-allowed ones (the launcher needs a verifiable hash to -// auto-install any mod, not only ranked-eligible ones), including +// exempt) and hashes each remaining mod's flattened, extracted content -- +// every mod, not just ranked-allowed ones (the launcher needs a verifiable +// hash to auto-install any mod, not only ranked-eligible ones), including // admin-created custom mods (listCustomMods() below), which aren't in the // fetched index at all -- that doesn't already have a stored hash for that -// exact version. "Prepared" means run through the same extract/flatten/rezip -// pipeline the launcher itself applies before deploying a mod into the Mods -// folder (see computePreparedZipHash's doc comment) -- not a hash of the raw +// exact version. Run through the same extract/flatten pipeline the launcher +// itself applies before deploying a mod into the Mods folder (see +// computeModFolderHashForRelease's doc comment) -- not a hash of the raw // download, which is never what actually gets loaded or what Ranked // verification checks against. A mod's hash is only ever recomputed when its // version changes. diff --git a/apps/server/src/tests/services/mod-folder-hash.test.ts b/apps/server/src/tests/services/mod-folder-hash.test.ts new file mode 100644 index 00000000..37bdcb53 --- /dev/null +++ b/apps/server/src/tests/services/mod-folder-hash.test.ts @@ -0,0 +1,88 @@ +import { createHash } from 'node:crypto' +import { promises as fs } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { computeModFolderHash } from '../../features/mods/mod-folder-hash.js' + +async function makeTree(root: string, files: Record): Promise { + for (const [relativePath, contents] of Object.entries(files)) { + const full = path.join(root, relativePath) + await fs.mkdir(path.dirname(full), { recursive: true }) + await fs.writeFile(full, contents) + } +} + +describe('computeModFolderHash', () => { + let tmpDir: string + + beforeEach(async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'folder-hash-test-')) + }) + + afterEach(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }) + }) + + it('matches a hand-computed sorted-path-plus-content sha256', async () => { + await makeTree(tmpDir, { + 'main.lua': 'return {}', + 'assets/icon.png': 'binary', + }) + + const expected = createHash('sha256') + .update('assets/icon.png', 'utf8') + .update(Buffer.from('binary')) + .update('main.lua', 'utf8') + .update(Buffer.from('return {}')) + .digest('hex') + + expect(await computeModFolderHash(tmpDir)).toBe(expected) + }) + + it('is independent of the root folder\'s own name', async () => { + await makeTree(tmpDir, { 'main.lua': 'return {}' }) + const first = await computeModFolderHash(tmpDir) + + const renamed = `${tmpDir}_renamed` + await fs.rename(tmpDir, renamed) + const second = await computeModFolderHash(renamed) + await fs.rename(renamed, tmpDir) // afterEach expects tmpDir to still exist + + expect(second).toBe(first) + }) + + it('is independent of filesystem enumeration order', async () => { + await makeTree(tmpDir, { + 'z_first.lua': 'a', + 'a_second.lua': 'b', + 'nested/deep/file.lua': 'c', + }) + + // Two independent walks of the same tree should always agree, + // regardless of whatever order readdir() happens to return. + const first = await computeModFolderHash(tmpDir) + const second = await computeModFolderHash(tmpDir) + expect(second).toBe(first) + }) + + it('changes if a file\'s content changes', async () => { + await makeTree(tmpDir, { 'main.lua': 'return {}' }) + const before = await computeModFolderHash(tmpDir) + + await fs.writeFile(path.join(tmpDir, 'main.lua'), 'return { changed = true }') + const after = await computeModFolderHash(tmpDir) + + expect(after).not.toBe(before) + }) + + it('changes if a file is renamed even with identical content', async () => { + await makeTree(tmpDir, { 'main.lua': 'return {}' }) + const before = await computeModFolderHash(tmpDir) + + await fs.rename(path.join(tmpDir, 'main.lua'), path.join(tmpDir, 'renamed.lua')) + const after = await computeModFolderHash(tmpDir) + + expect(after).not.toBe(before) + }) +})