Skip to content

PAT-1795 Bump staging-provider and job-queue-job-configuration #803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pepamartinec merged 3 commits into from
May 29, 2026
Merged

PAT-1795 Bump staging-provider and job-queue-job-configuration #803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4db2e07
feat(docker-bundle): thread staging workspace networkPolicy and bump …
pepamartinec May 27, 2026
5dcc722
ci(docker-bundle): use docker compose v2 in pipeline
pepamartinec May 27, 2026
d2a8fc9
test(docker-bundle): migrate test fixtures to developer-portal-v2 reg…
pepamartinec May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ QUAYIO_PRIVATE_USERNAME=keboola+dockerbundle_testsuite
QUAYIO_PRIVATE_PASSWORD=
AWS_ECR_ACCESS_KEY_ID=
AWS_ECR_SECRET_ACCESS_KEY=
AWS_ECR_REGISTRY_URI=061240556736.dkr.ecr.us-east-1.amazonaws.com/docker-testing
AWS_ECR_REGISTRY_ACCOUNT_ID=147946154733
AWS_ECR_REGISTRY_REGION=us-east-1
AWS_ECR_REGISTRY_ACCOUNT_ID=061240556736
AWS_ECR_REGISTRY_NAME=developer-portal-v2
AWS_KMS_TEST_KEY=alias/docker-runner-testing-docker-runner
AWS_KMS_REGION=eu-central-1
RUNNER_MIN_LOG_PORT=12202
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ STORAGE_API_TOKEN_NETWORK_POLICY=
# AWS ECR
# RepositoryUrl, Region — CloudFormation stack > Outputs tab
# IAM access keys — CloudFormation stack > Resources tab > DockerRunnerUser > IAM console > Security credentials > Create access key
AWS_ECR_REGISTRY_URI= # RepositoryUrl
AWS_ECR_REGISTRY_NAME= # ECR name (e.g. developer-portal-v2)
AWS_ECR_REGISTRY_REGION= # Region
AWS_ECR_REGISTRY_ACCOUNT_ID= # first part of the registry URI
AWS_ECR_ACCESS_KEY_ID= # IAM user access key
Expand Down
4 changes: 4 additions & 0 deletions Tests/BaseRunnerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
use Keboola\ObjectEncryptor\EncryptorOptions;
use Keboola\ObjectEncryptor\ObjectEncryptor;
use Keboola\ObjectEncryptor\ObjectEncryptorFactory;
use Keboola\StagingProvider\Workspace\Configuration\NetworkPolicy;
use Keboola\StorageApi\BranchAwareClient;
use Keboola\StorageApi\Client;
use Keboola\StorageApi\DevBranches;
Expand Down Expand Up @@ -198,6 +199,9 @@ protected function getRunner(?string $token = null): Runner
),
),
(int) self::getOptionalEnv('RUNNER_MIN_LOG_PORT'),
// tests run the component container externally (CI/local), so the ephemeral
// workspace must be reachable via the user network policy
stagingWorkspaceNetworkPolicy: NetworkPolicy::USER,
);
}

Expand Down
46 changes: 27 additions & 19 deletions Tests/Docker/Image/AWSElasticContainerRegistryTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,13 @@

class AWSElasticContainerRegistryTest extends BaseImageTest
{
private const TEST_IMAGE_TAG = '0.1.1';

private static function testImageUri(): string
{
return getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test';
}

public function testMissingCredentials(): void
{
putenv('AWS_ACCESS_KEY_ID=');
Expand Down Expand Up @@ -70,7 +77,7 @@ public function testImageNotFound(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'tag' => 'not-existing',
'repository' => [
'region' => getenv('AWS_ECR_REGISTRY_REGION'),
Expand All @@ -90,10 +97,10 @@ public function testImageNotFound(): void
public function testDownloadedImage()
{
Process::fromShellCommandline(
'sudo docker rmi -f $(sudo docker images -aq ' . getenv('AWS_ECR_REGISTRY_URI') . ')',
'sudo docker rmi -f $(sudo docker images -aq ' . self::testImageUri() . ')',
)->run();
$process = Process::fromShellCommandline(
'sudo docker images | grep ' . getenv('AWS_ECR_REGISTRY_URI') . ' | wc -l',
'sudo docker images | grep ' . self::testImageUri() . ' | wc -l',
);
$process->run();
self::assertEquals(0, trim($process->getOutput()));
Expand All @@ -102,7 +109,8 @@ public function testDownloadedImage()
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'tag' => self::TEST_IMAGE_TAG,
'repository' => [
'region' => getenv('AWS_ECR_REGISTRY_REGION'),
],
Expand All @@ -111,17 +119,17 @@ public function testDownloadedImage()
]);
$image = $this->imageFactory->getImage($imageConfig, true);
$image->prepare([]);
self::assertEquals(getenv('AWS_ECR_REGISTRY_URI') . ':latest', $image->getFullImageId());
$repoParts = explode('/', getenv('AWS_ECR_REGISTRY_URI'));
self::assertEquals(self::testImageUri() . ':' . self::TEST_IMAGE_TAG, $image->getFullImageId());
$repoParts = explode('/', self::testImageUri());
array_shift($repoParts);
self::assertEquals(implode('/', $repoParts) . ':latest', $image->getPrintableImageId());
self::assertEquals(implode('/', $repoParts) . ':' . self::TEST_IMAGE_TAG, $image->getPrintableImageId());

$process = Process::fromShellCommandline(
'sudo docker images | grep ' . getenv('AWS_ECR_REGISTRY_URI') . '| wc -l',
'sudo docker images | grep ' . self::testImageUri() . '| wc -l',
);
$process->run();
self::assertEquals(1, trim($process->getOutput()));
Process::fromShellCommandline('sudo docker rmi ' . getenv('AWS_ECR_REGISTRY_URI'));
Process::fromShellCommandline('sudo docker rmi ' . self::testImageUri());
}

public function testGetAwsAccountId(): void
Expand Down Expand Up @@ -168,37 +176,37 @@ public function testLogger(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'repository' => [
'region' => getenv('AWS_ECR_REGISTRY_REGION'),
],
'tag' => 'test-hash',
'tag' => self::TEST_IMAGE_TAG,
],
],
]);

$image = $this->imageFactory->getImage($imageConfig, true);
$image->prepare([]);

self::assertEquals(getenv('AWS_ECR_REGISTRY_URI') . ':test-hash', $image->getFullImageId());
$repoParts = explode('/', getenv('AWS_ECR_REGISTRY_URI'));
self::assertEquals(self::testImageUri() . ':' . self::TEST_IMAGE_TAG, $image->getFullImageId());
$repoParts = explode('/', self::testImageUri());
array_shift($repoParts);
self::assertEquals(implode('/', $repoParts) . ':test-hash', $image->getPrintableImageId());
self::assertEquals(implode('/', $repoParts) . ':' . self::TEST_IMAGE_TAG, $image->getPrintableImageId());
self::assertTrue($this->logsHandler->hasNotice(
'Using image ' . getenv('AWS_ECR_REGISTRY_URI') .
':test-hash with repo-digest ' . getenv('AWS_ECR_REGISTRY_URI') . '@sha256:' .
'Using image ' . self::testImageUri() .
':' . self::TEST_IMAGE_TAG . ' with repo-digest ' . self::testImageUri() . '@sha256:' .
ImageTest::TEST_HASH_DIGEST,
));
self::assertEquals(
[getenv('AWS_ECR_REGISTRY_URI') . '@sha256:' . ImageTest::TEST_HASH_DIGEST],
[self::testImageUri() . '@sha256:' . ImageTest::TEST_HASH_DIGEST],
$image->getImageDigests(),
);
$repoParts = explode('/', getenv('AWS_ECR_REGISTRY_URI'));
$repoParts = explode('/', self::testImageUri());
array_shift($repoParts);
self::assertEquals(
[implode('/', $repoParts) . '@sha256:' . ImageTest::TEST_HASH_DIGEST],
$image->getPrintableImageDigests(),
);
Process::fromShellCommandline('sudo docker rmi ' . getenv('AWS_ECR_REGISTRY_URI'))->run();
Process::fromShellCommandline('sudo docker rmi ' . self::testImageUri())->run();
}
}
4 changes: 2 additions & 2 deletions Tests/Docker/Image/ReplicatedRegistryImageTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

class ReplicatedRegistryImageTest extends TestCase
{
private const ECR_URI = '147946154733.dkr.ecr.us-east-1.amazonaws.com/keboola/test-component';
private const ECR_URI = '147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/test-component';
private const IMAGE_NAME = 'keboola/test-component';
private const REPLICATED_REGISTRY_URL = 'us-docker.pkg.dev/my-project/my-repo';

Expand Down Expand Up @@ -125,7 +125,7 @@ public function testGetImageIdFallsBackToUriTransformationWhenDefinitionNameMiss
$result = $image->getImageId();

self::assertSame(
'us-docker.pkg.dev/my-project/my-repo/keboola/test-component',
'us-docker.pkg.dev/my-project/my-repo/developer-portal-v2/test-component',
$result,
);
}
Expand Down
2 changes: 1 addition & 1 deletion Tests/Docker/ImageFactoryTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

class ImageFactoryTest extends TestCase
{
private const ECR_URI = '147946154733.dkr.ecr.us-east-1.amazonaws.com/keboola/test-component';
private const ECR_URI = '147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/test-component';
private const REPLICATED_REGISTRY_URL = 'us-docker.pkg.dev/my-project/my-repo';

public function testCreatesReplicatedRegistryImageWhenEnabled(): void
Expand Down
45 changes: 27 additions & 18 deletions Tests/Docker/ImageTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@

class ImageTest extends BaseImageTest
{
public const TEST_HASH_DIGEST = 'a89486bee7cadd59a966500cd837e0cea70a7989de52636652ae9fccfc958c9a';
public const TEST_HASH_DIGEST = 'f44de9927422695f478e6a9713f2dd21f6951b6f7cddbdb10500c2b720137042';
private const TEST_IMAGE_TAG = '0.1.1';

private static function testImageUri(): string
{
return getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test';
}

public function testDockerHub()
{
Expand Down Expand Up @@ -51,24 +57,26 @@ public function testQuayIO()

public function testImageDigestNotPulled()
{
$command = Process::fromShellCommandline('sudo docker rmi ' . getenv('AWS_ECR_REGISTRY_URI') . ':test-hash');
$command = Process::fromShellCommandline(
'sudo docker rmi ' . self::testImageUri() . ':' . self::TEST_IMAGE_TAG,
);
$command->run();
$imageConfig = new ComponentSpecification([
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'digest' => self::TEST_HASH_DIGEST,
'tag' => 'test-hash',
'tag' => self::TEST_IMAGE_TAG,
],
],
]);

$image = $this->imageFactory->getImage($imageConfig, true);
$image->prepare([]);
self::assertTrue($this->logsHandler->hasNoticeThatContains(
'Digest "a89486bee7cadd59a966500cd837e0cea70a7989de52636652ae9fccfc958c9a" for image ' .
'"' . getenv('AWS_ECR_REGISTRY_URI') .':test-hash" not found.',
'Digest "' . self::TEST_HASH_DIGEST . '" for image ' .
'"' . self::testImageUri() . ':' . self::TEST_IMAGE_TAG . '" not found.',
));
}

Expand All @@ -78,9 +86,9 @@ public function testImageDigestPulled()
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'digest' => self::TEST_HASH_DIGEST,
'tag' => 'test-hash',
'tag' => self::TEST_IMAGE_TAG,
],
],
]);
Expand All @@ -93,34 +101,35 @@ public function testImageDigestPulled()
$image = $this->imageFactory->getImage($imageConfig, true);
$image->prepare([]);
self::assertFalse($this->logsHandler->hasNoticeThatContains(
'Digest "a89486bee7cadd59a966500cd837e0cea70a7989de52636652ae9fccfc958c9a" for image ' .
'"' . getenv('AWS_ECR_REGISTRY_URI') .':test-hash" not found.',
'Digest "' . self::TEST_HASH_DIGEST . '" for image ' .
'"' . self::testImageUri() . ':' . self::TEST_IMAGE_TAG . '" not found.',
));
}

public function testImageDigestInvalid()
{
$imageConfig = new ComponentSpecification([
// Pull a different image to obtain a real digest that does not belong to the tested tag.
$otherImageConfig = new ComponentSpecification([
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-workspace-test',
'digest' => self::TEST_HASH_DIGEST,
'tag' => 'latest',
],
],
]);
$image = $this->imageFactory->getImage($imageConfig, true);
$image = $this->imageFactory->getImage($otherImageConfig, true);
$image->prepare([]);
preg_match('#@sha256:(.*)$#', $image->getImageDigests()[0], $matches);
self::assertCount(2, $matches);
$imageConfig = new ComponentSpecification([
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'digest' => $matches[1],
'tag' => 'test-hash',
'tag' => self::TEST_IMAGE_TAG,
],
],
]);
Expand All @@ -132,7 +141,7 @@ public function testImageDigestInvalid()
$image->prepare([]);
self::assertTrue($this->logsHandler->hasNoticeThatContains(
'Digest "' . $matches[1] . '" for image ' .
'"' . getenv('AWS_ECR_REGISTRY_URI') .':test-hash" not found.',
'"' . self::testImageUri() . ':' . self::TEST_IMAGE_TAG . '" not found.',
));
}

Expand Down Expand Up @@ -187,9 +196,9 @@ public function testGetProcessTimeout(
'process_timeout' => $componentTimeout,
'definition' => [
'type' => 'aws-ecr',
'uri' => getenv('AWS_ECR_REGISTRY_URI'),
'uri' => self::testImageUri(),
'digest' => self::TEST_HASH_DIGEST,
'tag' => 'test-hash',
'tag' => self::TEST_IMAGE_TAG,
],
],
]),
Expand Down
24 changes: 12 additions & 12 deletions Tests/Runner/RunnerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3183,7 +3183,7 @@ public function testWorkspaceMapping(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-workspace-test',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-workspace-test',
'tag' => 'latest',
],
'staging_storage' => [
Expand Down Expand Up @@ -3266,7 +3266,7 @@ public function testWorkspaceMappingCleanupComponentError(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-workspace-test',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-workspace-test',
'tag' => 'latest',
],
'staging_storage' => [
Expand Down Expand Up @@ -3344,7 +3344,7 @@ public function testWorkspaceMappingCleanupMappingError(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-workspace-test',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-workspace-test',
'tag' => 'latest',
],
'staging_storage' => [
Expand Down Expand Up @@ -3433,8 +3433,8 @@ public function testS3StagingMapping(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-staging-test',
'tag' => '0.0.3',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test',
'tag' => '0.1.1',
],
'staging_storage' => [
'input' => 's3',
Expand Down Expand Up @@ -3512,8 +3512,8 @@ public function testStorageFilesOutputProcessed(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-staging-test',
'tag' => '0.1.0',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test',
'tag' => '0.1.1',
],
'staging_storage' => [
'input' => 'local',
Expand Down Expand Up @@ -3596,8 +3596,8 @@ public function testOutputTablesAsFiles(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-staging-test',
'tag' => '0.1.0',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test',
'tag' => '0.1.1',
],
'staging_storage' => [
'input' => 'local',
Expand Down Expand Up @@ -3678,8 +3678,8 @@ public function testOutputTablesAsFilesWithMissingTableFiles(): void
'data' => [
'definition' => [
'type' => 'aws-ecr',
'uri' => '061240556736.dkr.ecr.us-east-1.amazonaws.com/keboola.runner-staging-test',
'tag' => '0.1.0',
'uri' => getenv('AWS_ECR_REGISTRY_URI') . '/keboola.runner-staging-test',
'tag' => '0.1.1',
],
'staging_storage' => [
'input' => 'local',
Expand Down Expand Up @@ -3831,7 +3831,7 @@ public function testOutputTablesOnJobFailureWorkspace(): void
'type' => 'aws-ecr',
// phpcs:ignore Generic.Files.LineLength.MaxExceeded
'uri' => '147946154733.dkr.ecr.us-east-1.amazonaws.com/developer-portal-v2/keboola.snowflake-transformation',
'tag' => '0.7.0',
'tag' => '1.5.0',
],
'staging_storage' => [
'input' => 'workspace-snowflake',
Expand Down
Loading