Buildbarn ci

.github/buildbarn/.bazelrc.template

@@ -0,0 +1,7 @@
+build --remote_executor=grpc://localhost:8980
+build --remote_timeout=3600
+build --spawn_strategy=remote,local
+build --strategy=CppCompile=remote
+build --remote_instance_name=fuse
+build --remote_default_exec_properties=OSFamily=linux
+build --remote_default_exec_properties=container-image=docker://ghcr.io/catthehacker/ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448

.github/buildbarn/cleanup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -eu
+
+worker_fuse="worker-fuse-ubuntu22-04"
+worker_hardlinking="worker-hardlinking-ubuntu22-04"
+fuse_dir_to_unmount="volumes/${worker_fuse}/build"
+sudo -v
+
+cleanup() {
+    EXIT_STATUS=$?
+    local -
+    set -x
+
+    sudo fusermount -u "$fuse_dir_to_unmount" || true
+    exit "$EXIT_STATUS"
+}
+
+cleanup

.github/buildbarn/config/common.libsonnet

@@ -0,0 +1,42 @@
+{
+  blobstore: {
+    contentAddressableStorage: {
+      sharding: {
+        shards: {
+          "0": {
+            backend: { grpc: { address: 'storage-0:8981' } },
+            weight: 1,
+          },
+        },
+      },
+    },
+    actionCache: {
+      completenessChecking: {
+        backend: {
+          sharding: {
+            shards: {
+              "0": {
+                backend: { grpc: { address: 'storage-0:8981' } },
+                weight: 1,
+              },
+            },
+          },
+        },
+        maximumTotalTreeSizeBytes: 64 * 1024 * 1024,
+      },
+    },
+  },
+  browserUrl: 'http://localhost:7984',
+  maximumMessageSizeBytes: 2 * 1024 * 1024,
+  global: {
+    diagnosticsHttpServer: {
+      httpServers: [{
+        listenAddresses: [':80'],
+        authenticationPolicy: { allow: {} },
+      }],
+      enablePrometheus: true,
+      enablePprof: true,
+      enableActiveSpans: true,
+    },
+  },
+}

.github/buildbarn/config/frontend.jsonnet

@@ -0,0 +1,36 @@
+local common = import 'common.libsonnet';
+
+{
+  grpcServers: [{
+    listenAddresses: [':8980'],
+    authenticationPolicy: { allow: {} },
+  }],
+  schedulers: {
+    '': {
+      endpoint: {
+        address: 'scheduler:8982',
+        addMetadataJmespathExpression: {
+          expression: |||
+            {
+              "build.bazel.remote.execution.v2.requestmetadata-bin": incomingGRPCMetadata."build.bazel.remote.execution.v2.requestmetadata-bin"
+            }
+          |||,
+        },
+      },
+    },
+  },
+  maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+  global: common.global,
+  contentAddressableStorage: {
+    backend: common.blobstore.contentAddressableStorage,
+    getAuthorizer: { allow: {} },
+    putAuthorizer: { allow: {} },
+    findMissingAuthorizer: { allow: {} },
+  },
+  actionCache: {
+    backend: common.blobstore.actionCache,
+    getAuthorizer: { allow: {} },
+    putAuthorizer: { allow: {} },
+  },
+  executeAuthorizer: { allow: {} },
+}

.github/buildbarn/config/runner-ubuntu22-04.jsonnet

@@ -0,0 +1,10 @@
+local common = import 'common.libsonnet';
+
+{
+  buildDirectoryPath: '/worker/build',
+  global: common.global,
+  grpcServers: [{
+    listenPaths: ['/worker/runner'],
+    authenticationPolicy: { allow: {} },
+  }],
+}

.github/buildbarn/config/scheduler.jsonnet

@@ -0,0 +1,42 @@
+local common = import 'common.libsonnet';
+
+{
+  adminHttpServers: [{
+    listenAddresses: [':7982'],
+    authenticationPolicy: { allow: {} },
+  }],
+  clientGrpcServers: [{
+    listenAddresses: [':8982'],
+    authenticationPolicy: { allow: {} },
+  }],
+  workerGrpcServers: [{
+    listenAddresses: [':8983'],
+    authenticationPolicy: { allow: {} },
+  }],
+  buildQueueStateGrpcServers: [{
+    listenAddresses: [':8984'],
+    authenticationPolicy: { allow: {} },
+  }],
+  browserUrl: common.browserUrl,
+  contentAddressableStorage: common.blobstore.contentAddressableStorage,
+  maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+  global: common.global,
+  executeAuthorizer: { allow: {} },
+  modifyDrainsAuthorizer: { allow: {} },
+  killOperationsAuthorizer: { allow: {} },
+  synchronizeAuthorizer: { allow: {} },
+  actionRouter: {
+    simple: {
+      platformKeyExtractor: { action: {} },
+      invocationKeyExtractors: [
+        { correlatedInvocationsId: {} },
+        { toolInvocationId: {} },
+      ],
+      initialSizeClassAnalyzer: {
+        defaultExecutionTimeout: '1800s',
+        maximumExecutionTimeout: '7200s',
+      },
+    },
+  },
+  platformQueueWithNoWorkersTimeout: '900s',
+}

.github/buildbarn/config/storage.jsonnet

@@ -0,0 +1,75 @@
+local common = import 'common.libsonnet';
+
+{
+  grpcServers: [{
+    listenAddresses: [':8981'],
+    authenticationPolicy: { allow: {} },
+  }],
+  maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+  global: common.global,
+  contentAddressableStorage: {
+    backend: {
+      'local': {
+        keyLocationMapOnBlockDevice: {
+          file: {
+            path: '/storage-cas/key_location_map',
+            sizeBytes: 400 * 1024 * 1024,
+          },
+        },
+        keyLocationMapMaximumGetAttempts: 16,
+        keyLocationMapMaximumPutAttempts: 64,
+        oldBlocks: 8,
+        currentBlocks: 24,
+        newBlocks: 3,
+        blocksOnBlockDevice: {
+          source: {
+            file: {
+              path: '/storage-cas/blocks',
+              sizeBytes: 32 * 1024 * 1024 * 1024,
+            },
+          },
+          spareBlocks: 3,
+        },
+        persistent: {
+          stateDirectoryPath: '/storage-cas/persistent_state',
+          minimumEpochInterval: '300s',
+        },
+      },
+    },
+    getAuthorizer: { allow: {} },
+    putAuthorizer: { allow: {} },
+    findMissingAuthorizer: { allow: {} },
+  },
+  actionCache: {
+    backend: {
+      'local': {
+        keyLocationMapOnBlockDevice: {
+          file: {
+            path: '/storage-ac/key_location_map',
+            sizeBytes: 1024 * 1024,
+          },
+        },
+        keyLocationMapMaximumGetAttempts: 16,
+        keyLocationMapMaximumPutAttempts: 64,
+        oldBlocks: 8,
+        currentBlocks: 24,
+        newBlocks: 1,
+        blocksOnBlockDevice: {
+          source: {
+            file: {
+              path: '/storage-ac/blocks',
+              sizeBytes: 20 * 1024 * 1024,
+            },
+          },
+          spareBlocks: 3,
+        },
+        persistent: {
+          stateDirectoryPath: '/storage-ac/persistent_state',
+          minimumEpochInterval: '300s',
+        },
+      },
+    },
+    getAuthorizer: { allow: {} },
+    putAuthorizer: { allow: {} },
+  },
+}

.github/buildbarn/config/worker-fuse-ubuntu22-04.jsonnet

@@ -0,0 +1,103 @@
+local common = import 'common.libsonnet';
+
+// The FUSE worker is the most efficient configuration.
+// This is preferred to the hardlinking configuration.
+{
+  blobstore: {
+    actionCache: common.blobstore.actionCache,
+    contentAddressableStorage: {
+      readCaching: {
+        slow: common.blobstore.contentAddressableStorage,
+        fast: {
+          'local': {
+            keyLocationMapOnBlockDevice: {
+              file: {
+                path: '/worker/cas/key_location_map',
+                sizeBytes: 400 * 1024 * 1024,
+              },
+            },
+            keyLocationMapMaximumGetAttempts: 16,
+            keyLocationMapMaximumPutAttempts: 64,
+            oldBlocks: 8,
+            currentBlocks: 24,
+            newBlocks: 3,
+            blocksOnBlockDevice: {
+              source: {
+                file: {
+                  path: '/worker/cas/blocks',
+                  sizeBytes: 32 * 1024 * 1024 * 1024,
+                },
+              },
+              spareBlocks: 3,
+              dataIntegrityValidationCache: {
+                cacheSize: 50000,
+                cacheDuration: '14400s',
+                cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+              },
+            },
+            persistent: {
+              stateDirectoryPath: '/worker/cas/persistent_state',
+              minimumEpochInterval: '300s',
+            },
+          },
+        },
+        replicator: { deduplicating: { 'local': {} } },
+      },
+    },
+  },
+  browserUrl: common.browserUrl,
+  maximumMessageSizeBytes: common.maximumMessageSizeBytes,
+  scheduler: { address: 'scheduler:8983' },
+  global: common.global,
+  buildDirectories: [{
+    virtual: {
+      maximumExecutionTimeoutCompensation: '3600s',
+      shuffleDirectoryListings: true,
+      maximumWritableFileUploadDelay: '60s',
+      mount: {
+        mountPath: '/worker/build',
+        fuse: {
+          directoryEntryValidity: '300s',
+          inodeAttributeValidity: '300s',
+          allowOther: true,
+          mountMethod: 'DIRECT',
+        },
+      },
+    },
+    runners: [{
+      endpoint: { address: 'unix:///worker/runner' },
+      concurrency: 8,
+      instanceNamePrefix: 'fuse',
+      platform: {
+        properties: [
+          { name: 'OSFamily', value: 'linux' },
+          { name: 'container-image', value: 'docker://ghcr.io/catthehacker/ubuntu:act-22.04@sha256:5f9c35c25db1d51a8ddaae5c0ba8d3c163c5e9a4a6cc97acd409ac7eae239448' },
+        ],
+      },
+      maximumFilePoolFileCount: 100000,
+      maximumFilePoolSizeBytes: 1 * 1024 * 1024 * 1024,
+      workerId: {
+        datacenter: 'amsterdam',
+        rack: '3',
+        slot: '10',
+        hostname: 'ubuntu-worker.example.com',
+      },
+    }],
+  }],
+  filePool: {
+    blockDevice: {
+      file: {
+        path: '/worker/filepool',
+        // concurrency * maximumFilePoolSizeBytes
+        sizeBytes: 8 * 1024 * 1024 * 1024,
+      },
+    },
+  },
+  inputDownloadConcurrency: 10,
+  outputUploadConcurrency: 11,
+  directoryCache: {
+    maximumCount: 1000,
+    maximumSizeBytes: 1000 * 1024,
+    cacheReplacementPolicy: 'LEAST_RECENTLY_USED',
+  },
+}