Add invariant tests

acceptance/bin/check_plan.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""
+Check that all actions in plan are "skip".
+"""
+
+import sys
+import json
+import pprint
+
+
+def check_plan(path):
+    with open(path) as fobj:
+        raw = fobj.read()
+
+    changes_detected = 0
+
+    try:
+        data = json.loads(raw)
+        for key, value in data["plan"].items():
+            if value["action"] != "skip":
+                print(f"Unexpected {action=} for {key}")
+                changes_detected += 1
+    except Exception:
+        print(raw, flush=True)
+        raise
+
+    if changes_detected:
+        print(raw, flush=True)
+        sys.exit(10)
+
+
+def main():
+    for path in sys.argv[1:]:
+        check_plan(path)
+
+
+if __name__ == "__main__":
+    main()

acceptance/invariant/README.md

@@ -0,0 +1,6 @@
+Invariant tests are acceptance tests that can be run against many configs to check for certain properties.
+Unlike regular acceptance tests full output is not recorded, unless the condition is not met. For example,
+no_drift test checks that there are no actions planned after successful deploy. If that's not the case, the
+test will dump full JSON plan to the output.
+
+In order to add a new test, add a config to configs/ and include it in test.toml.

acceptance/invariant/configs/alert.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  alerts:
+    foo:
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      display_name: test-alert-$UNIQUE_NAME
+      file_path: ./alert.dbalert.json

acceptance/invariant/configs/app.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  apps:
+    foo:
+      name: app-$UNIQUE_NAME
+      source_code_path: ./app

acceptance/invariant/configs/cluster.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  clusters:
+    foo:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: 13.3.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1

acceptance/invariant/configs/dashboard.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  dashboards:
+    foo:
+      warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
+      display_name: test-dashboard-$UNIQUE_NAME
+      file_path: ./dashboard.lvdash.json

acceptance/invariant/configs/database_catalog.yml.tmpl

@@ -0,0 +1,15 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  database_instances:
+    instance1:
+      name: test-db-instance-$UNIQUE_NAME
+      capacity: CU_1
+
+  database_catalogs:
+    foo:
+      database_instance_name: ${resources.database_instances.instance1.name}
+      database_name: test_db
+      name: test-catalog-$UNIQUE_NAME
+      create_database_if_not_exists: true

acceptance/invariant/configs/database_instance.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  database_instances:
+    foo:
+      name: test-db-instance-$UNIQUE_NAME
+      capacity: CU_1

acceptance/invariant/configs/experiment.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  experiments:
+    foo:
+      name: /Users/$CURRENT_USER_NAME/test-experiment-$UNIQUE_NAME

acceptance/invariant/configs/job.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    foo:
+      name: test-job-$UNIQUE_NAME

acceptance/invariant/configs/model.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  models:
+    foo:
+      name: test-model-$UNIQUE_NAME

acceptance/invariant/configs/model_serving_endpoint.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  model_serving_endpoints:
+    foo:
+      name: test-endpoint-$UNIQUE_NAME

acceptance/invariant/configs/pipeline.yml.tmpl

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    foo:
+      name: test-pipeline-$UNIQUE_NAME
+      libraries:
+        - file:
+            path: pipeline.py

acceptance/invariant/configs/registered_model.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  registered_models:
+    foo:
+      name: test-model-$UNIQUE_NAME
+      catalog_name: main
+      schema_name: default

acceptance/invariant/configs/schema.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  schemas:
+    foo:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME

acceptance/invariant/configs/secret_scope.yml.tmpl

@@ -0,0 +1,8 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  secret_scopes:
+    foo:
+      name: test-scope-$UNIQUE_NAME
+      backend_type: DATABRICKS

acceptance/invariant/configs/sql_warehouse.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  sql_warehouses:
+    foo:
+      name: test-warehouse-$UNIQUE_NAME
+      cluster_size: X-Small
+      max_num_clusters: 1

acceptance/invariant/configs/synced_database_table.yml.tmpl

@@ -0,0 +1,26 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  database_instances:
+    instance1:
+      name: test-db-instance-$UNIQUE_NAME
+      capacity: CU_1
+
+  database_catalogs:
+    catalog1:
+      database_instance_name: ${resources.database_instances.instance1.name}
+      database_name: test_db
+      name: test-catalog-$UNIQUE_NAME
+      create_database_if_not_exists: true
+
+  synced_database_tables:
+    foo:
+      name: ${resources.database_catalogs.catalog1.name}.${resources.database_catalogs.catalog1.database_name}.test_table
+      database_instance_name: ${resources.database_instances.instance1.name}
+      logical_database_name: ${resources.database_catalogs.catalog1.database_name}
+      spec:
+        source_table_full_name: samples.nyctaxi.trips
+        scheduling_policy: SNAPSHOT
+        primary_key_columns:
+          - tpep_pickup_datetime

acceptance/invariant/configs/volume.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  volumes:
+    foo:
+      name: test-volume-$UNIQUE_NAME
+      catalog_name: main
+      schema_name: default

acceptance/invariant/data/alert.dbalert.json

@@ -0,0 +1,27 @@
+{
+  "custom_summary": "test-alert",
+  "evaluation": {
+    "source": {
+      "name": "1",
+      "display": "1",
+      "aggregation": "MAX"
+    },
+    "comparison_operator": "EQUAL",
+    "threshold": {
+      "value": {
+        "double_value": 1.0
+      }
+    },
+    "notification": {
+      "retrigger_seconds": 60,
+      "notify_on_ok": false
+    }
+  },
+  "schedule": {
+    "quartz_cron_schedule": "0 0 * * * ?",
+    "timezone_id": "UTC"
+  },
+  "query_lines": [
+    "select 1"
+  ]
+}

acceptance/invariant/data/app/app.py

@@ -0,0 +1,8 @@
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def hello():
+    return "Hello"

acceptance/invariant/data/app/app.yaml

@@ -0,0 +1 @@
+command: ["flask", "--app", "app", "run"]

acceptance/invariant/data/dashboard.lvdash.json

@@ -0,0 +1 @@
+{"pages":[{"name":"page1","displayName":"Test Page"}]}

acceptance/invariant/data/pipeline.py

@@ -0,0 +1,6 @@
+import dlt
+
+
+@dlt.table
+def my_table():
+    return spark.range(1)

acceptance/invariant/no_drift/output.txt

@@ -0,0 +1 @@
+INPUT_CONFIG_OK

acceptance/invariant/no_drift/script

@@ -0,0 +1,38 @@
+# Invariant to test: no drift after deploy
+# Additional checks: no internal errors / panics in validate/plan/deploy
+
+# Copy data files to test directory
+cp -r $TESTDIR/../data/* . 2>/dev/null || true
+
+envsubst < $TESTDIR/../configs/$INPUT_CONFIG > databricks.yml
+
+cp databricks.yml LOG.config
+
+# We redirect output rather than record it because some configs that are being tested may produce warnings
+trace $CLI bundle validate &> LOG.validate
+
+cat LOG.validate | contains.py '!panic' '!internal error' > /dev/null
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve &> LOG.destroy
+    cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null
+}
+
+trap cleanup EXIT
+
+trace $CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null
+
+# Special message to fuzzer that generated config was fine.
+# Any failures after this point will be considered as "bug detected" by fuzzer.
+echo INPUT_CONFIG_OK
+
+$CLI bundle plan -o json &> plan.json
+cat plan.json | contains.py '!panic' '!internal error' > /dev/null
+
+# Check both text and JSON plan for no changes
+# Note, expect that there maybe more than one resource unchanged
+$CLI bundle plan | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+
+$CLI bundle plan -o json > plan.json
+check_plan.py plan.json

acceptance/invariant/test.toml

@@ -0,0 +1,36 @@
+Local = true
+Cloud = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+    "plan.json",
+    "*.py",
+    "*.json",
+    "app",
+]
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [
+    "direct",
+    # These tests work on terraform but commented out to save time on test run
+    #"terraform",
+]
+
+EnvMatrix.INPUT_CONFIG = [
+    "alert.yml.tmpl",
+    "app.yml.tmpl",
+    "cluster.yml.tmpl",
+    "dashboard.yml.tmpl",
+    "database_catalog.yml.tmpl",
+    "database_instance.yml.tmpl",
+    "experiment.yml.tmpl",
+    "job.yml.tmpl",
+    "model.yml.tmpl",
+    "model_serving_endpoint.yml.tmpl",
+    "pipeline.yml.tmpl",
+    "registered_model.yml.tmpl",
+    "schema.yml.tmpl",
+    "secret_scope.yml.tmpl",
+    "synced_database_table.yml.tmpl",
+    "volume.yml.tmpl",
+]