diff --git a/.circleci/config.yml b/.circleci/config.yml index 4143cad465b..f50bcd7b73a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -622,10 +622,9 @@ jobs: key: react-spectrum-{{ .Environment.CACHE_VERSION }}-{{ .Environment.CIRCLE_SHA1 }} - run: - name: run verdaccio and publish built packages + name: build packages and seed verdaccio storage directly command: | mkdir -p verdaccio-workspace - ./scripts/verdaccio-ci.sh ./scripts/verdaccio-deploy.sh environment: VERDACCIO_STORAGE_PATH: /tmp/verdaccio-workspace/storage diff --git a/scripts/verdaccio-deploy.sh b/scripts/verdaccio-deploy.sh index 69182be24b1..f6e1ce07517 100755 --- a/scripts/verdaccio-deploy.sh +++ b/scripts/verdaccio-deploy.sh @@ -1,31 +1,15 @@ #!/usr/bin/env bash -port=4000 -# usually defaults to https://registry.npmjs.com/ -original_registry=`npm get registry` -registry="http://localhost:$port" -output="output.out" -touch $output +# Builds all packages, bumps their versions, and writes them straight into a +# Verdaccio local-storage folder set -e -echo "Build and deploy to verdaccio" +# Where the Verdaccio local-storage database is written. Must match the storage +# path used by the downstream verdaccio-ci.sh / verdaccio-config.yaml jobs. +storage_path="${VERDACCIO_STORAGE_PATH:-/tmp/verdaccio-workspace/storage}" -# Wait for verdaccio to start -grep -q 'http address' <(tail -f $output) - -if curl -sI http://localhost:4000/ >/dev/null; then - echo "Verdaccio is running on port 4000." -else - echo "Verdaccio is NOT running on port 4000." -fi - -yarn config set npmPublishRegistry $registry -yarn config set npmRegistryServer $registry -yarn config set npmAlwaysAuth false -yarn config set npmAuthToken abc -yarn config set unsafeHttpWhitelist localhost -npm set registry $registry +echo "Build and seed Verdaccio storage at $storage_path" git config --global user.email octobot@github.com git config --global user.name GitHub Actions @@ -39,9 +23,6 @@ cat .yarn/versions/version.yml yarn version apply --all cat ./packages/react-aria-components/package.json -# Publish packages to verdaccio -yarn workspaces foreach --all --no-private -pt npm publish --tag latest - -curl -s http://localhost:4000/@adobe/react-spectrum - -netstat -tpln | awk -F'[[:space:]/:]+' '$5 == 4000 {print $(NF-2)}' | xargs kill +# Seed the packages directly into Verdaccio storage +mkdir -p "$storage_path" +yarn workspaces list --json --no-private | node ./scripts/verdaccio-seed.js "$storage_path" diff --git a/scripts/verdaccio-seed.js b/scripts/verdaccio-seed.js new file mode 100644 index 00000000000..b8a5c777a6e --- /dev/null +++ b/scripts/verdaccio-seed.js @@ -0,0 +1,144 @@ +#!/usr/bin/env node +'use strict'; + +// Seeds a Verdaccio 6 local-storage folder directly, without running the Verdaccio +// server or issuing `npm publish` for each package. +// +// Storage layout (validated against verdaccio@6.0.5 / @verdaccio/local-storage-legacy): +// /.verdaccio-db.json { list: [...names], secret } +// //-.tgz +// //package.json the verdaccio "packument" +// +// Usage: +// yarn workspaces list --json --no-private | node ./scripts/verdaccio-seed.js [storagePath] +// +// storagePath defaults to $VERDACCIO_STORAGE_PATH or /tmp/verdaccio-workspace/storage. + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const packlist = require('npm-packlist'); +const tar = require('tar'); + +const REGISTRY = process.env.SEED_REGISTRY || 'http://localhost:4000'; +const CONCURRENCY = Number(process.env.SEED_CONCURRENCY) || 16; + +function unscoped(name) { + return name.startsWith('@') ? name.slice(name.indexOf('/') + 1) : name; +} + +function tarballBuffer(dir, files) { + return new Promise((resolve, reject) => { + const chunks = []; + tar + .create({cwd: dir, gzip: true, portable: true, prefix: 'package/'}, files) + .on('data', c => chunks.push(c)) + .on('end', () => resolve(Buffer.concat(chunks))) + .on('error', reject); + }); +} + +async function seedOne(storagePath, location) { + const dir = path.resolve(process.cwd(), location); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf8')); + if (manifest.private) { + return null; + } + const {name, version} = manifest; + if (!name || !version) { + throw new Error(`Workspace at ${location} is missing name or version`); + } + + const files = await packlist({path: dir}); + const buf = await tarballBuffer(dir, files); + + const shasum = crypto.createHash('sha1').update(buf).digest('hex'); + const integrity = 'sha512-' + crypto.createHash('sha512').update(buf).digest('base64'); + const tgzName = `${unscoped(name)}-${version}.tgz`; + + const pkgDir = path.join(storagePath, name); + fs.mkdirSync(pkgDir, {recursive: true}); + fs.writeFileSync(path.join(pkgDir, tgzName), buf); + + const now = new Date().toISOString(); + const versionManifest = Object.assign({}, manifest, { + _id: `${name}@${version}`, + dist: {integrity, shasum, tarball: `${REGISTRY}/${name}/-/${tgzName}`} + }); + const packument = { + name, + versions: {[version]: versionManifest}, + time: {modified: now, created: now, [version]: now}, + users: {}, + 'dist-tags': {latest: version}, + _uplinks: {}, + _distfiles: {}, + _attachments: {[tgzName]: {shasum, version}}, + _rev: '1-' + crypto.randomBytes(8).toString('hex'), + _id: name, + readme: manifest.readme || 'ERROR: No README data found!' + }; + fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify(packument)); + return name; +} + +async function readWorkspaceLocations() { + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + return chunks + .join('') + .trim() + .split('\n') + .filter(Boolean) + .map(line => JSON.parse(line)) + .filter(ws => ws.name && ws.location && ws.location !== '.') + .map(ws => ws.location); +} + +async function pool(items, worker, size) { + const results = []; + let index = 0; + async function run() { + while (index < items.length) { + const i = index++; + results[i] = await worker(items[i]); + } + } + await Promise.all(Array.from({length: Math.min(size, items.length)}, run)); + return results; +} + +async function main() { + const storagePath = path.resolve( + process.argv[2] || process.env.VERDACCIO_STORAGE_PATH || '/tmp/verdaccio-workspace/storage' + ); + fs.mkdirSync(storagePath, {recursive: true}); + + const locations = await readWorkspaceLocations(); + const start = Date.now(); + const seeded = await pool(locations, loc => seedOne(storagePath, loc), CONCURRENCY); + const names = seeded.filter(Boolean); + + const dbPath = path.join(storagePath, '.verdaccio-db.json'); + let db = {list: [], secret: crypto.randomBytes(16).toString('hex')}; + if (fs.existsSync(dbPath)) { + try { + db = JSON.parse(fs.readFileSync(dbPath, 'utf8')); + } catch (e) { + // corrupt/empty db, start fresh + } + } + db.list = Array.from(new Set([...(db.list || []), ...names])); + fs.writeFileSync(dbPath, JSON.stringify(db)); + + console.log( + `Seeded ${names.length} packages into ${storagePath} in ${((Date.now() - start) / 1000).toFixed(1)}s` + ); +} + +main().catch(err => { + console.error(err); + process.exit(1); +});