diff --git a/acceptance/bin/check_plan.py b/acceptance/bin/check_plan.py new file mode 100755 index 0000000000..eeb1480690 --- /dev/null +++ b/acceptance/bin/check_plan.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +Check that all actions in plan are "skip". +""" + +import sys +import json +import pprint + + +def check_plan(path): + with open(path) as fobj: + raw = fobj.read() + + changes_detected = 0 + + try: + data = json.loads(raw) + for key, value in data["plan"].items(): + if value["action"] != "skip": + print(f"Unexpected {action=} for {key}") + changes_detected += 1 + except Exception: + print(raw, flush=True) + raise + + if changes_detected: + print(raw, flush=True) + sys.exit(10) + + +def main(): + for path in sys.argv[1:]: + check_plan(path) + + +if __name__ == "__main__": + main() diff --git a/acceptance/invariant/README.md b/acceptance/invariant/README.md new file mode 100644 index 0000000000..184d3f541c --- /dev/null +++ b/acceptance/invariant/README.md @@ -0,0 +1,6 @@ +Invariant tests are acceptance tests that can be run against many configs to check for certain properties. +Unlike regular acceptance tests full output is not recorded, unless the condition is not met. For example, +no_drift test checks that there are no actions planned after successful deploy. If that's not the case, the +test will dump full JSON plan to the output. + +In order to add a new test, add a config to configs/ and include it in test.toml. diff --git a/acceptance/invariant/configs/alert.yml.tmpl b/acceptance/invariant/configs/alert.yml.tmpl new file mode 100644 index 0000000000..ea4acf7ffa --- /dev/null +++ b/acceptance/invariant/configs/alert.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + alerts: + foo: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + display_name: test-alert-$UNIQUE_NAME + file_path: ./alert.dbalert.json diff --git a/acceptance/invariant/configs/app.yml.tmpl b/acceptance/invariant/configs/app.yml.tmpl new file mode 100644 index 0000000000..7506f8fdc9 --- /dev/null +++ b/acceptance/invariant/configs/app.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + apps: + foo: + name: app-$UNIQUE_NAME + source_code_path: ./app diff --git a/acceptance/invariant/configs/cluster.yml.tmpl b/acceptance/invariant/configs/cluster.yml.tmpl new file mode 100644 index 0000000000..aece5a3619 --- /dev/null +++ b/acceptance/invariant/configs/cluster.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + clusters: + foo: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 diff --git a/acceptance/invariant/configs/dashboard.yml.tmpl b/acceptance/invariant/configs/dashboard.yml.tmpl new file mode 100644 index 0000000000..a8a6d95ba1 --- /dev/null +++ b/acceptance/invariant/configs/dashboard.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + dashboards: + foo: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + display_name: test-dashboard-$UNIQUE_NAME + file_path: ./dashboard.lvdash.json diff --git a/acceptance/invariant/configs/database_catalog.yml.tmpl b/acceptance/invariant/configs/database_catalog.yml.tmpl new file mode 100644 index 0000000000..78d20eee42 --- /dev/null +++ b/acceptance/invariant/configs/database_catalog.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + database_instances: + instance1: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 + + database_catalogs: + foo: + database_instance_name: ${resources.database_instances.instance1.name} + database_name: test_db + name: test-catalog-$UNIQUE_NAME + create_database_if_not_exists: true diff --git a/acceptance/invariant/configs/database_instance.yml.tmpl b/acceptance/invariant/configs/database_instance.yml.tmpl new file mode 100644 index 0000000000..1498593596 --- /dev/null +++ b/acceptance/invariant/configs/database_instance.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + database_instances: + foo: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 diff --git a/acceptance/invariant/configs/experiment.yml.tmpl b/acceptance/invariant/configs/experiment.yml.tmpl new file mode 100644 index 0000000000..58a442e523 --- /dev/null +++ b/acceptance/invariant/configs/experiment.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + experiments: + foo: + name: /Users/$CURRENT_USER_NAME/test-experiment-$UNIQUE_NAME diff --git a/acceptance/invariant/configs/job.yml.tmpl b/acceptance/invariant/configs/job.yml.tmpl new file mode 100644 index 0000000000..c94cc5bc8b --- /dev/null +++ b/acceptance/invariant/configs/job.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + foo: + name: test-job-$UNIQUE_NAME diff --git a/acceptance/invariant/configs/model.yml.tmpl b/acceptance/invariant/configs/model.yml.tmpl new file mode 100644 index 0000000000..e105a731a3 --- /dev/null +++ b/acceptance/invariant/configs/model.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + models: + foo: + name: test-model-$UNIQUE_NAME diff --git a/acceptance/invariant/configs/model_serving_endpoint.yml.tmpl b/acceptance/invariant/configs/model_serving_endpoint.yml.tmpl new file mode 100644 index 0000000000..102f14c049 --- /dev/null +++ b/acceptance/invariant/configs/model_serving_endpoint.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + model_serving_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME diff --git a/acceptance/invariant/configs/pipeline.yml.tmpl b/acceptance/invariant/configs/pipeline.yml.tmpl new file mode 100644 index 0000000000..8e412adbcd --- /dev/null +++ b/acceptance/invariant/configs/pipeline.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + pipelines: + foo: + name: test-pipeline-$UNIQUE_NAME + libraries: + - file: + path: pipeline.py diff --git a/acceptance/invariant/configs/registered_model.yml.tmpl b/acceptance/invariant/configs/registered_model.yml.tmpl new file mode 100644 index 0000000000..5f605085d7 --- /dev/null +++ b/acceptance/invariant/configs/registered_model.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + registered_models: + foo: + name: test-model-$UNIQUE_NAME + catalog_name: main + schema_name: default diff --git a/acceptance/invariant/configs/schema.yml.tmpl b/acceptance/invariant/configs/schema.yml.tmpl new file mode 100644 index 0000000000..658dce88dd --- /dev/null +++ b/acceptance/invariant/configs/schema.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + schemas: + foo: + catalog_name: main + name: test-schema-$UNIQUE_NAME diff --git a/acceptance/invariant/configs/secret_scope.yml.tmpl b/acceptance/invariant/configs/secret_scope.yml.tmpl new file mode 100644 index 0000000000..e891ad0000 --- /dev/null +++ b/acceptance/invariant/configs/secret_scope.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + secret_scopes: + foo: + name: test-scope-$UNIQUE_NAME + backend_type: DATABRICKS diff --git a/acceptance/invariant/configs/sql_warehouse.yml.tmpl b/acceptance/invariant/configs/sql_warehouse.yml.tmpl new file mode 100644 index 0000000000..3fefbdaa35 --- /dev/null +++ b/acceptance/invariant/configs/sql_warehouse.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + sql_warehouses: + foo: + name: test-warehouse-$UNIQUE_NAME + cluster_size: X-Small + max_num_clusters: 1 diff --git a/acceptance/invariant/configs/synced_database_table.yml.tmpl b/acceptance/invariant/configs/synced_database_table.yml.tmpl new file mode 100644 index 0000000000..ebb8161dc2 --- /dev/null +++ b/acceptance/invariant/configs/synced_database_table.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + database_instances: + instance1: + name: test-db-instance-$UNIQUE_NAME + capacity: CU_1 + + database_catalogs: + catalog1: + database_instance_name: ${resources.database_instances.instance1.name} + database_name: test_db + name: test-catalog-$UNIQUE_NAME + create_database_if_not_exists: true + + synced_database_tables: + foo: + name: ${resources.database_catalogs.catalog1.name}.${resources.database_catalogs.catalog1.database_name}.test_table + database_instance_name: ${resources.database_instances.instance1.name} + logical_database_name: ${resources.database_catalogs.catalog1.database_name} + spec: + source_table_full_name: samples.nyctaxi.trips + scheduling_policy: SNAPSHOT + primary_key_columns: + - tpep_pickup_datetime diff --git a/acceptance/invariant/configs/volume.yml.tmpl b/acceptance/invariant/configs/volume.yml.tmpl new file mode 100644 index 0000000000..69d462b942 --- /dev/null +++ b/acceptance/invariant/configs/volume.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + volumes: + foo: + name: test-volume-$UNIQUE_NAME + catalog_name: main + schema_name: default diff --git a/acceptance/invariant/data/alert.dbalert.json b/acceptance/invariant/data/alert.dbalert.json new file mode 100644 index 0000000000..2b78b24c90 --- /dev/null +++ b/acceptance/invariant/data/alert.dbalert.json @@ -0,0 +1,27 @@ +{ + "custom_summary": "test-alert", + "evaluation": { + "source": { + "name": "1", + "display": "1", + "aggregation": "MAX" + }, + "comparison_operator": "EQUAL", + "threshold": { + "value": { + "double_value": 1.0 + } + }, + "notification": { + "retrigger_seconds": 60, + "notify_on_ok": false + } + }, + "schedule": { + "quartz_cron_schedule": "0 0 * * * ?", + "timezone_id": "UTC" + }, + "query_lines": [ + "select 1" + ] +} diff --git a/acceptance/invariant/data/app/app.py b/acceptance/invariant/data/app/app.py new file mode 100644 index 0000000000..5d37802a07 --- /dev/null +++ b/acceptance/invariant/data/app/app.py @@ -0,0 +1,8 @@ +from flask import Flask + +app = Flask(__name__) + + +@app.route("/") +def hello(): + return "Hello" diff --git a/acceptance/invariant/data/app/app.yaml b/acceptance/invariant/data/app/app.yaml new file mode 100644 index 0000000000..489ab6586c --- /dev/null +++ b/acceptance/invariant/data/app/app.yaml @@ -0,0 +1 @@ +command: ["flask", "--app", "app", "run"] diff --git a/acceptance/invariant/data/dashboard.lvdash.json b/acceptance/invariant/data/dashboard.lvdash.json new file mode 100644 index 0000000000..4b8560b160 --- /dev/null +++ b/acceptance/invariant/data/dashboard.lvdash.json @@ -0,0 +1 @@ +{"pages":[{"name":"page1","displayName":"Test Page"}]} diff --git a/acceptance/invariant/data/pipeline.py b/acceptance/invariant/data/pipeline.py new file mode 100644 index 0000000000..e1b4968c11 --- /dev/null +++ b/acceptance/invariant/data/pipeline.py @@ -0,0 +1,6 @@ +import dlt + + +@dlt.table +def my_table(): + return spark.range(1) diff --git a/acceptance/invariant/no_drift/out.test.toml b/acceptance/invariant/no_drift/out.test.toml new file mode 100644 index 0000000000..4f2b1ccc28 --- /dev/null +++ b/acceptance/invariant/no_drift/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "job.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/invariant/no_drift/output.txt b/acceptance/invariant/no_drift/output.txt new file mode 100644 index 0000000000..7a28cb73a5 --- /dev/null +++ b/acceptance/invariant/no_drift/output.txt @@ -0,0 +1 @@ +INPUT_CONFIG_OK diff --git a/acceptance/invariant/no_drift/script b/acceptance/invariant/no_drift/script new file mode 100644 index 0000000000..db80c73cdc --- /dev/null +++ b/acceptance/invariant/no_drift/script @@ -0,0 +1,38 @@ +# Invariant to test: no drift after deploy +# Additional checks: no internal errors / panics in validate/plan/deploy + +# Copy data files to test directory +cp -r $TESTDIR/../data/* . 2>/dev/null || true + +envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml + +cp databricks.yml LOG.config + +# We redirect output rather than record it because some configs that are being tested may produce warnings +trace $CLI bundle validate &> LOG.validate + +cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null + +cleanup() { + trace $CLI bundle destroy --auto-approve &> LOG.destroy + cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null +} + +trap cleanup EXIT + +trace $CLI bundle deploy &> LOG.deploy +cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null + +# Special message to fuzzer that generated config was fine. +# Any failures after this point will be considered as "bug detected" by fuzzer. +echo INPUT_CONFIG_OK + +$CLI bundle plan -o json &> plan.json +cat plan.json | contains.py '!panic' '!internal error' > /dev/null + +# Check both text and JSON plan for no changes +# Note, expect that there maybe more than one resource unchanged +$CLI bundle plan | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan + +$CLI bundle plan -o json > plan.json +check_plan.py plan.json diff --git a/acceptance/invariant/test.toml b/acceptance/invariant/test.toml new file mode 100644 index 0000000000..80a8f50378 --- /dev/null +++ b/acceptance/invariant/test.toml @@ -0,0 +1,36 @@ +Local = true +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", + "plan.json", + "*.py", + "*.json", + "app", +] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [ + "direct", + # These tests work on terraform but commented out to save time on test run + #"terraform", +] + +EnvMatrix.INPUT_CONFIG = [ + "alert.yml.tmpl", + "app.yml.tmpl", + "cluster.yml.tmpl", + "dashboard.yml.tmpl", + "database_catalog.yml.tmpl", + "database_instance.yml.tmpl", + "experiment.yml.tmpl", + "job.yml.tmpl", + "model.yml.tmpl", + "model_serving_endpoint.yml.tmpl", + "pipeline.yml.tmpl", + "registered_model.yml.tmpl", + "schema.yml.tmpl", + "secret_scope.yml.tmpl", + "synced_database_table.yml.tmpl", + "volume.yml.tmpl", +]