Skip to content

Commit a118aa4

Browse files
mrahanjammrahanjam
committed
add e2e tests for dataplex metadata update
1 parent 097fd84 commit a118aa4

13 files changed

Lines changed: 453 additions & 1 deletion

File tree

‎.github/workflows/e2e.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ jobs:
4040
)
4141
strategy:
4242
matrix:
43-
tests: [bigquery, common, gcs, pubsub, spanner, gcsdelete, gcsmove, bigqueryexecute]
43+
tests: [bigquery, common, gcs, pubsub, spanner, gcsdelete, gcsmove, bigqueryexecute, dataplex]
4444
fail-fast: false
4545
steps:
4646
# Pinned 1.0.0 version

‎src/e2e-test/features/dataplex/sink/BigQueryToDataplex.feature‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
@Dataplex_Sink
2+
Feature: Dataplex sink - Verification of BQ source to dataplex sink with update metadata enabled
3+
4+
@Dataplex_SINK_TEST @GCS_SINK_TEST @BQ_SOURCE_TEST @Dataplex_Sink_Required
5+
Scenario:Validate successful records transfer from BigQuery to Dataplex with update metadata enabled
6+
Given Open Datafusion Project to configure pipeline
7+
When Source is BigQuery
8+
When Sink is Dataplex
9+
Then Connect source as "BigQuery" and sink as "Dataplex" to establish connection
10+
Then Open BigQuery source properties
11+
Then Enter BigQuery property reference name
12+
Then Enter BigQuery property projectId "projectId"
13+
Then Enter BigQuery property datasetProjectId "projectId"
14+
Then Enter BigQuery property dataset "dataset"
15+
Then Enter BigQuery source property table name
16+
Then Override Service account details if set in environment variables
17+
Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled
18+
Then Validate output schema with expectedSchema "bqSourceSchema"
19+
Then Validate "BigQuery" plugin properties
20+
Then Close the BigQuery properties
21+
Then Open Dataplex sink properties
22+
Then Override Service account details if set in environment variables
23+
Then Enter the Dataplex mandatory properties
24+
Then Enter the Dataplex sink mandatory properties
25+
Then Enable Metadata Update
26+
Then Validate "Dataplex" plugin properties
27+
Then Close the Dataplex properties
28+
Then Preview and run the pipeline
29+
Then Wait till pipeline preview is in running state
30+
Then Open and capture pipeline preview logs
31+
Then Verify the preview run status of pipeline in the logs is "succeeded"
32+
Then Close the pipeline logs
33+
Then Close the preview
34+
Then Save and Deploy Pipeline
35+
Then Run the Pipeline in Runtime
36+
Then Wait till pipeline is in running state
37+
Then Open and capture logs
38+
Then Verify the pipeline status is "Succeeded"

‎src/e2e-test/features/dataplex/sink/DataplexSourceToDataplexSink.feature‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
@Dataplex_Sink
2+
Feature: Dataplex sink - Verification of BQ source to dataplex sink with update metadata enabled
3+
4+
@Dataplex_SINK_TEST @GCS_SINK_TEST @BQ_SOURCE_TEST
5+
Scenario:Validate successful records transfer from BigQuery to Dataplex with update metadata enabled
6+
Given Open Datafusion Project to configure pipeline
7+
When Source is BigQuery
8+
When Sink is Dataplex
9+
Then Connect source as "BigQuery" and sink as "Dataplex" to establish connection
10+
Then Open BigQuery source properties
11+
Then Enter BigQuery property reference name
12+
Then Enter BigQuery property projectId "projectId"
13+
Then Enter BigQuery property datasetProjectId "projectId"
14+
Then Enter BigQuery property dataset "dataset"
15+
Then Enter BigQuery source property table name
16+
Then Override Service account details if set in environment variables
17+
Then Enter BiqQuery property encryption key name "cmekBQ" if cmek is enabled
18+
Then Validate output schema with expectedSchema "bqSourceSchema"
19+
Then Validate "BigQuery" plugin properties
20+
Then Close the BigQuery properties
21+
Then Open Dataplex sink properties
22+
Then Override Service account details if set in environment variables
23+
Then Enter the Dataplex mandatory properties
24+
Then Enter the Dataplex sink mandatory properties
25+
Then Enable Metadata Update
26+
Then Validate "Dataplex" plugin properties
27+
Then Close the Dataplex properties
28+
Then Save and Deploy Pipeline
29+
Then Run the Pipeline in Runtime
30+
Then Wait till pipeline is in running state
31+
Then Open and capture logs
32+
Then Verify the pipeline status is "Succeeded"
33+
Given Open Datafusion Project to configure pipeline
34+
When Source is Dataplex
35+
When Sink is Dataplex
36+
Then Connect source as "Dataplex" and sink as "Dataplex" to establish connection
37+
Then Open Dataplex source properties
38+
Then Enter the Dataplex mandatory properties
39+
Then Enter the Dataplex source mandatory properties
40+
Then Override Service account details if set in environment variables
41+
Then Validate output schema with expectedSchema "dataplexSourceSchema"
42+
Then Validate "Dataplex" plugin properties
43+
Then Close the Dataplex properties
44+
Then Open Dataplex sink properties
45+
Then Override Service account details if set in environment variables
46+
Then Enter the Dataplex mandatory properties
47+
Then Enter the Dataplex sink mandatory properties
48+
Then Enable Metadata Update
49+
Then Remove "ts" column from output schema
50+
Then Validate "Dataplex" plugin properties
51+
Then Close the Dataplex properties
52+
Then Save and Deploy Pipeline
53+
Then Run the Pipeline in Runtime
54+
Then Wait till pipeline is in running state
55+
Then Open and capture logs
56+
Then Verify the pipeline status is "Succeeded"

‎src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunner.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.dataplex.runners.sinkrunner;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
23+
/**
24+
* Test Runner to execute Dataplex sink cases.
25+
*/
26+
@RunWith(Cucumber.class)
27+
@CucumberOptions(
28+
features = {"src/e2e-test/features"},
29+
glue = {"io.cdap.plugin.dataplex.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
30+
"io.cdap.plugin.bigquery.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
31+
tags = {"@Dataplex_Sink and not @ignore"},
32+
monochrome = true,
33+
plugin = {"pretty", "html:target/cucumber-html-report/dataplex-sink",
34+
"json:target/cucumber-reports/cucumber-dataplex-sink.json",
35+
"junit:target/cucumber-reports/cucumber-dataplex-sink.xml"}
36+
)
37+
public class TestRunner {
38+
}

‎src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/TestRunnerRequired.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.dataplex.runners.sinkrunner;
17+
18+
import io.cucumber.junit.Cucumber;
19+
import io.cucumber.junit.CucumberOptions;
20+
import org.junit.runner.RunWith;
21+
22+
/**
23+
* Test Runner to execute only required dataplex Sink plugin test cases.
24+
*/
25+
@RunWith(Cucumber.class)
26+
@CucumberOptions(
27+
features = {"src/e2e-test/features"},
28+
glue = {"io.cdap.plugin.dataplex.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
29+
"io.cdap.plugin.bigquery.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
30+
tags = {"@Dataplex_Sink_Required and not @ignore"},
31+
monochrome = true,
32+
plugin = {"pretty", "html:target/cucumber-html-report/dataplex-sink-required",
33+
"json:target/cucumber-reports/cucumber-dataplex-sink-required.json",
34+
"junit:target/cucumber-reports/cucumber-dataplex-sink-required.xml"}
35+
)
36+
public class TestRunnerRequired {
37+
}

‎src/e2e-test/java/io/cdap/plugin/dataplex/runners/sinkrunner/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* Package contains the runner for the Dataplex sink features.
19+
*/
20+
package io.cdap.plugin.dataplex.runners.sinkrunner;

‎src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexBase.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.dataplex.stepsdesign;
17+
18+
import io.cdap.e2e.utils.PluginPropertyUtils;
19+
import io.cdap.plugin.utils.DataplexHelper;
20+
import io.cdap.plugin.utils.E2EHelper;
21+
import io.cucumber.java.en.Then;
22+
23+
import java.io.IOException;
24+
25+
/**
26+
* Dataplex related common stepDesigns.
27+
*/
28+
public class DataplexBase implements E2EHelper {
29+
@Then("Enter the Dataplex mandatory properties")
30+
public void enterTheDataplexMandatoryProperties() throws IOException {
31+
DataplexHelper.enterReferenceName();
32+
DataplexHelper.enterProjectId();
33+
DataplexHelper.enterDataplexProperty(
34+
"location", PluginPropertyUtils.pluginProp("dataplexDefaultLocation"));
35+
DataplexHelper.enterDataplexProperty("lake", PluginPropertyUtils.pluginProp("dataplexDefaultLake"));
36+
DataplexHelper.enterDataplexProperty("zone", PluginPropertyUtils.pluginProp("dataplexDefaultZone"));
37+
}
38+
}

‎src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSink.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.dataplex.stepsdesign;
17+
18+
import io.cdap.e2e.pages.actions.CdfPluginPropertiesActions;
19+
import io.cdap.e2e.pages.actions.CdfStudioActions;
20+
import io.cdap.e2e.utils.PluginPropertyUtils;
21+
import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
22+
import io.cdap.plugin.utils.DataplexHelper;
23+
import io.cdap.plugin.utils.E2EHelper;
24+
import io.cucumber.java.en.Then;
25+
import io.cucumber.java.en.When;
26+
27+
import java.io.IOException;
28+
29+
30+
/**
31+
* Dataplex Sink related stepDesigns.
32+
*/
33+
public class DataplexSink implements E2EHelper {
34+
35+
@When("Sink is Dataplex")
36+
public void sinkIsDataplex() {
37+
CdfStudioActions.clickSink();
38+
selectSinkPlugin("Dataplex");
39+
}
40+
41+
@Then("Open Dataplex sink properties")
42+
public void openDataplexSinkProperties() {
43+
openSinkPluginProperties("Dataplex");
44+
}
45+
46+
@Then("Close the Dataplex properties")
47+
public void closeTheBigQueryProperties() {
48+
CdfPluginPropertiesActions.clickCloseButton();
49+
}
50+
51+
@Then("Enter the Dataplex sink mandatory properties")
52+
public void enterTheDataplexSinkMandatoryProperties() throws IOException {
53+
DataplexHelper.enterDataplexProperty("asset", PluginPropertyUtils.pluginProp("dataplexDefaultAsset"));
54+
DataplexHelper.setAssetType("STORAGE_BUCKET");
55+
DataplexHelper.enterDataplexProperty("table", TestSetupHooks.gcsTargetBucketName);
56+
}
57+
58+
@Then("Enable Metadata Update")
59+
public void enableMetadataUpdate() {
60+
DataplexHelper.toggleMetadataUpdate();
61+
}
62+
63+
@Then("Remove {string} column from output schema")
64+
public void removeColumnFromOutputSchema(String fieldName) throws InterruptedException {
65+
DataplexHelper.deleteSchemaField(fieldName);
66+
}
67+
}

‎src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/DataplexSource.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
package io.cdap.plugin.dataplex.stepsdesign;
17+
18+
import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
19+
import io.cdap.plugin.utils.DataplexHelper;
20+
import io.cdap.plugin.utils.E2EHelper;
21+
import io.cucumber.java.en.Then;
22+
import io.cucumber.java.en.When;
23+
24+
import java.io.IOException;
25+
26+
/**
27+
* Dataplex Source related stepDesigns.
28+
*/
29+
public class DataplexSource implements E2EHelper {
30+
31+
@When("Source is Dataplex")
32+
public void sourceIsDataplex() {
33+
selectSourcePlugin("Dataplex");
34+
}
35+
36+
@Then("Open Dataplex source properties")
37+
public void openDataplexSourceProperties() {
38+
openSourcePluginProperties("Dataplex");
39+
}
40+
41+
@Then("Enter the Dataplex source mandatory properties")
42+
public void enterTheDataplexSourceMandatoryProperties() throws IOException {
43+
DataplexHelper.enterDataplexProperty(
44+
"entity", TestSetupHooks.gcsTargetBucketName.replaceAll("[^a-zA-Z0-9_]", "_"));
45+
}
46+
}

‎src/e2e-test/java/io/cdap/plugin/dataplex/stepsdesign/package-info.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright © 2022 Cask Data, Inc.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
5+
* use this file except in compliance with the License. You may obtain a copy of
6+
* the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13+
* License for the specific language governing permissions and limitations under
14+
* the License.
15+
*/
16+
17+
/**
18+
* This package contains stepDesigns for Dataplex Plugin.
19+
*/
20+
package io.cdap.plugin.dataplex.stepsdesign;

0 commit comments

Comments
 (0)