chore: Script the generate of Verdaccio's database to skip slow publish

.circleci/config.yml

@@ -622,10 +622,9 @@ jobs:
           key: react-spectrum-{{ .Environment.CACHE_VERSION }}-{{ .Environment.CIRCLE_SHA1 }}
 
       - run:
-          name: run verdaccio and publish built packages
+          name: build packages and seed verdaccio storage directly
           command: |
             mkdir -p verdaccio-workspace
-            ./scripts/verdaccio-ci.sh
             ./scripts/verdaccio-deploy.sh
           environment:
             VERDACCIO_STORAGE_PATH: /tmp/verdaccio-workspace/storage

scripts/verdaccio-deploy.sh

@@ -1,31 +1,15 @@
 #!/usr/bin/env bash
 
-port=4000
-# usually defaults to https://registry.npmjs.com/
-original_registry=`npm get registry`
-registry="http://localhost:$port"
-output="output.out"
-touch $output
+# Builds all packages, bumps their versions, and writes them straight into a
+# Verdaccio local-storage folder
 
 set -e
 
-echo "Build and deploy to verdaccio"
+# Where the Verdaccio local-storage database is written. Must match the storage
+# path used by the downstream verdaccio-ci.sh / verdaccio-config.yaml jobs.
+storage_path="${VERDACCIO_STORAGE_PATH:-/tmp/verdaccio-workspace/storage}"
 
-# Wait for verdaccio to start
-grep -q 'http address' <(tail -f $output)
-
-if curl -sI http://localhost:4000/ >/dev/null; then
-    echo "Verdaccio is running on port 4000."
-else
-    echo "Verdaccio is NOT running on port 4000."
-fi
-
-yarn config set npmPublishRegistry $registry
-yarn config set npmRegistryServer $registry
-yarn config set npmAlwaysAuth false
-yarn config set npmAuthToken abc
-yarn config set unsafeHttpWhitelist localhost
-npm set registry $registry
+echo "Build and seed Verdaccio storage at $storage_path"
 
 git config --global user.email octobot@github.com
 git config --global user.name GitHub Actions
@@ -39,9 +23,6 @@ cat .yarn/versions/version.yml
 yarn version apply --all
 cat ./packages/react-aria-components/package.json
 
-# Publish packages to verdaccio
-yarn workspaces foreach --all --no-private -pt npm publish --tag latest
-
-curl -s http://localhost:4000/@adobe/react-spectrum
-
-netstat -tpln | awk -F'[[:space:]/:]+' '$5 == 4000 {print $(NF-2)}' | xargs kill
+# Seed the packages directly into Verdaccio storage
+mkdir -p "$storage_path"
+yarn workspaces list --json --no-private | node ./scripts/verdaccio-seed.js "$storage_path"

scripts/verdaccio-seed.js

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+'use strict';
+
+// Seeds a Verdaccio 6 local-storage folder directly, without running the Verdaccio
+// server or issuing `npm publish` for each package.
+//
+// Storage layout (validated against verdaccio@6.0.5 / @verdaccio/local-storage-legacy):
+//   <storage>/.verdaccio-db.json          { list: [...names], secret }
+//   <storage>/<name>/<unscopedName>-<version>.tgz
+//   <storage>/<name>/package.json         the verdaccio "packument"
+//
+// Usage:
+//   yarn workspaces list --json --no-private | node ./scripts/verdaccio-seed.js [storagePath]
+//
+// storagePath defaults to $VERDACCIO_STORAGE_PATH or /tmp/verdaccio-workspace/storage.
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const packlist = require('npm-packlist');
+const tar = require('tar');
+
+const REGISTRY = process.env.SEED_REGISTRY || 'http://localhost:4000';
+const CONCURRENCY = Number(process.env.SEED_CONCURRENCY) || 16;
+
+function unscoped(name) {
+  return name.startsWith('@') ? name.slice(name.indexOf('/') + 1) : name;
+}
+
+function tarballBuffer(dir, files) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    tar
+      .create({cwd: dir, gzip: true, portable: true, prefix: 'package/'}, files)
+      .on('data', c => chunks.push(c))
+      .on('end', () => resolve(Buffer.concat(chunks)))
+      .on('error', reject);
+  });
+}
+
+async function seedOne(storagePath, location) {
+  const dir = path.resolve(process.cwd(), location);
+  const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8'));
+  if (manifest.private) {
+    return null;
+  }
+  const {name, version} = manifest;
+  if (!name || !version) {
+    throw new Error(`Workspace at ${location} is missing name or version`);
+  }
+
+  const files = await packlist({path: dir});
+  const buf = await tarballBuffer(dir, files);
+
+  const shasum = crypto.createHash('sha1').update(buf).digest('hex');
+  const integrity = 'sha512-' + crypto.createHash('sha512').update(buf).digest('base64');
+  const tgzName = `${unscoped(name)}-${version}.tgz`;
+
+  const pkgDir = path.join(storagePath, name);
+  fs.mkdirSync(pkgDir, {recursive: true});
+  fs.writeFileSync(path.join(pkgDir, tgzName), buf);
+
+  const now = new Date().toISOString();
+  const versionManifest = Object.assign({}, manifest, {
+    _id: `${name}@${version}`,
+    dist: {integrity, shasum, tarball: `${REGISTRY}/${name}/-/${tgzName}`}
+  });
+  const packument = {
+    name,
+    versions: {[version]: versionManifest},
+    time: {modified: now, created: now, [version]: now},
+    users: {},
+    'dist-tags': {latest: version},
+    _uplinks: {},
+    _distfiles: {},
+    _attachments: {[tgzName]: {shasum, version}},
+    _rev: '1-' + crypto.randomBytes(8).toString('hex'),
+    _id: name,
+    readme: manifest.readme || 'ERROR: No README data found!'
+  };
+  fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify(packument));
+  return name;
+}
+
+async function readWorkspaceLocations() {
+  const chunks = [];
+  for await (const chunk of process.stdin) {
+    chunks.push(chunk);
+  }
+  return chunks
+    .join('')
+    .trim()
+    .split('\n')
+    .filter(Boolean)
+    .map(line => JSON.parse(line))
+    .filter(ws => ws.name && ws.location && ws.location !== '.')
+    .map(ws => ws.location);
+}
+
+async function pool(items, worker, size) {
+  const results = [];
+  let index = 0;
+  async function run() {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await worker(items[i]);
+    }
+  }
+  await Promise.all(Array.from({length: Math.min(size, items.length)}, run));
+  return results;
+}
+
+async function main() {
+  const storagePath = path.resolve(
+    process.argv[2] || process.env.VERDACCIO_STORAGE_PATH || '/tmp/verdaccio-workspace/storage'
+  );
+  fs.mkdirSync(storagePath, {recursive: true});
+
+  const locations = await readWorkspaceLocations();
+  const start = Date.now();
+  const seeded = await pool(locations, loc => seedOne(storagePath, loc), CONCURRENCY);
+  const names = seeded.filter(Boolean);
+
+  const dbPath = path.join(storagePath, '.verdaccio-db.json');
+  let db = {list: [], secret: crypto.randomBytes(16).toString('hex')};
+  if (fs.existsSync(dbPath)) {
+    try {
+      db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
+    } catch (e) {
+      // corrupt/empty db, start fresh
+    }
+  }
+  db.list = Array.from(new Set([...(db.list || []), ...names]));
+  fs.writeFileSync(dbPath, JSON.stringify(db));
+
+  console.log(
+    `Seeded ${names.length} packages into ${storagePath} in ${((Date.now() - start) / 1000).toFixed(1)}s`
+  );
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});