Skip to content

Commit 1c1b2b5

Browse files
priyabhatnagar25priyabhatnagar25
committed
e2e BQ test scenarios new feature
1 parent ac5a3d7 commit 1c1b2b5

7 files changed

Lines changed: 425 additions & 8 deletions

File tree

‎src/e2e-test/features/bigquery/sink/BigQueryToBigQuerySink.feature‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,3 +195,154 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
195195
Then Open and capture logs
196196
Then Verify the pipeline status is "Succeeded"
197197
Then Verify the partition table is created with partitioned on field "bqPartitionFieldTime"
198+
199+
@BQ_SOURCE_DATATYPE_TEST @BQ_EXISTING_SINK_TEST @EXISTING_BQ_CONNECTION
200+
Scenario: Validate user is able to read the records from BigQuery source(existing table),source table here has more columns than BigQuery sink(existing table) with update button schema and truncate table to true with use connection functionality
201+
Given Open Datafusion Project to configure pipeline
202+
When Expand Plugin group in the LHS plugins list: "Source"
203+
When Select plugin: "BigQuery" from the plugins list as: "Source"
204+
When Expand Plugin group in the LHS plugins list: "Sink"
205+
When Select plugin: "BigQuery" from the plugins list as: "Sink"
206+
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
207+
Then Navigate to the properties page of plugin: "BigQuery"
208+
Then Click plugin property: "switch-useConnection"
209+
Then Click on the Browse Connections button
210+
Then Select connection: "bqConnectionName"
211+
Then Click on the Browse button inside plugin properties
212+
Then Select connection data row with name: "dataset"
213+
Then Select connection data row with name: "bqSourceTable"
214+
Then Wait till connection data loading completes with a timeout of 60 seconds
215+
Then Verify input plugin property: "dataset" contains value: "dataset"
216+
Then Verify input plugin property: "table" contains value: "bqSourceTable"
217+
Then Click on the Get Schema button
218+
Then Validate "BigQuery" plugin properties
219+
And Close the Plugin Properties page
220+
Then Navigate to the properties page of plugin: "BigQuery2"
221+
Then Click plugin property: "useConnection"
222+
Then Click on the Browse Connections button
223+
Then Select connection: "bqConnectionName"
224+
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
225+
Then Click on the Browse button inside plugin properties
226+
Then Click SELECT button inside connection data row with name: "dataset"
227+
Then Wait till connection data loading completes with a timeout of 60 seconds
228+
Then Verify input plugin property: "dataset" contains value: "dataset"
229+
Then Enter input plugin property: "table" with value: "bqTargetTable"
230+
Then Click plugin property: "updateTableSchema"
231+
Then Click plugin property: "truncateTable"
232+
Then Validate "BigQuery" plugin properties
233+
Then Close the BigQuery properties
234+
Then Save the pipeline
235+
Then Preview and run the pipeline
236+
Then Wait till pipeline preview is in running state
237+
Then Open and capture pipeline preview logs
238+
Then Verify the preview run status of pipeline in the logs is "succeeded"
239+
Then Close the pipeline logs
240+
Then Close the preview
241+
Then Deploy the pipeline
242+
Then Run the Pipeline in Runtime
243+
Then Wait till pipeline is in running state
244+
Then Open and capture logs
245+
Then Verify the pipeline status is "Succeeded"
246+
Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table
247+
248+
@BQ_INSERT_SOURCE_TEST @BQ_UPDATE_SINK_TEST @EXISTING_BQ_CONNECTION
249+
Scenario:Validate successful records transfer from BigQuery to BigQuery with Advanced Operations Update without updating the destination table schema with use connection functionality
250+
Given Open Datafusion Project to configure pipeline
251+
When Expand Plugin group in the LHS plugins list: "Source"
252+
When Select plugin: "BigQuery" from the plugins list as: "Source"
253+
When Expand Plugin group in the LHS plugins list: "Sink"
254+
When Select plugin: "BigQuery" from the plugins list as: "Sink"
255+
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
256+
Then Navigate to the properties page of plugin: "BigQuery"
257+
Then Click plugin property: "switch-useConnection"
258+
Then Click on the Browse Connections button
259+
Then Select connection: "bqConnectionName"
260+
Then Click on the Browse button inside plugin properties
261+
Then Select connection data row with name: "dataset"
262+
Then Select connection data row with name: "bqSourceTable"
263+
Then Wait till connection data loading completes with a timeout of 60 seconds
264+
Then Verify input plugin property: "dataset" contains value: "dataset"
265+
Then Verify input plugin property: "table" contains value: "bqSourceTable"
266+
Then Click on the Get Schema button
267+
Then Validate "BigQuery" plugin properties
268+
And Close the Plugin Properties page
269+
Then Navigate to the properties page of plugin: "BigQuery2"
270+
Then Click plugin property: "useConnection"
271+
Then Click on the Browse Connections button
272+
Then Select connection: "bqConnectionName"
273+
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
274+
Then Click on the Browse button inside plugin properties
275+
Then Click SELECT button inside connection data row with name: "dataset"
276+
Then Wait till connection data loading completes with a timeout of 60 seconds
277+
Then Verify input plugin property: "dataset" contains value: "dataset"
278+
Then Enter input plugin property: "table" with value: "bqTargetTable"
279+
And Select radio button plugin property: "operation" with value: "update"
280+
Then Click on the Add Button of the property: "relationTableKey" with value:
281+
| TableKey |
282+
Then Validate "BigQuery" plugin properties
283+
And Close the Plugin Properties page
284+
Then Save the pipeline
285+
Then Preview and run the pipeline
286+
Then Wait till pipeline preview is in running state
287+
Then Open and capture pipeline preview logs
288+
Then Verify the preview run status of pipeline in the logs is "succeeded"
289+
Then Close the pipeline logs
290+
Then Close the preview
291+
Then Deploy the pipeline
292+
Then Run the Pipeline in Runtime
293+
Then Wait till pipeline is in running state
294+
Then Open and capture logs
295+
Then Close the pipeline logs
296+
Then Verify the pipeline status is "Succeeded"
297+
Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table
298+
299+
@BQ_INSERT_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
300+
Scenario:Validate successful records transfer from BigQuery to BigQuery with Advanced operations Upsert without updating the destination table schema with use connection functionality
301+
Given Open Datafusion Project to configure pipeline
302+
When Expand Plugin group in the LHS plugins list: "Source"
303+
When Select plugin: "BigQuery" from the plugins list as: "Source"
304+
When Expand Plugin group in the LHS plugins list: "Sink"
305+
When Select plugin: "BigQuery" from the plugins list as: "Sink"
306+
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
307+
Then Navigate to the properties page of plugin: "BigQuery"
308+
Then Click plugin property: "switch-useConnection"
309+
Then Click on the Browse Connections button
310+
Then Select connection: "bqConnectionName"
311+
Then Click on the Browse button inside plugin properties
312+
Then Select connection data row with name: "dataset"
313+
Then Select connection data row with name: "bqSourceTable"
314+
Then Wait till connection data loading completes with a timeout of 60 seconds
315+
Then Verify input plugin property: "dataset" contains value: "dataset"
316+
Then Verify input plugin property: "table" contains value: "bqSourceTable"
317+
Then Click on the Get Schema button
318+
Then Validate "BigQuery" plugin properties
319+
And Close the Plugin Properties page
320+
Then Navigate to the properties page of plugin: "BigQuery2"
321+
Then Click plugin property: "useConnection"
322+
Then Click on the Browse Connections button
323+
Then Select connection: "bqConnectionName"
324+
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
325+
Then Click on the Browse button inside plugin properties
326+
Then Click SELECT button inside connection data row with name: "dataset"
327+
Then Wait till connection data loading completes with a timeout of 60 seconds
328+
Then Verify input plugin property: "dataset" contains value: "dataset"
329+
Then Enter input plugin property: "table" with value: "bqTargetTable"
330+
And Select radio button plugin property: "operation" with value: "upsert"
331+
Then Click on the Add Button of the property: "relationTableKey" with value:
332+
| TableKey |
333+
Then Validate "BigQuery" plugin properties
334+
And Close the Plugin Properties page
335+
Then Save the pipeline
336+
Then Preview and run the pipeline
337+
Then Wait till pipeline preview is in running state
338+
Then Open and capture pipeline preview logs
339+
Then Verify the preview run status of pipeline in the logs is "succeeded"
340+
Then Close the pipeline logs
341+
Then Close the preview
342+
Then Deploy the pipeline
343+
Then Run the Pipeline in Runtime
344+
Then Wait till pipeline is in running state
345+
Then Open and capture logs
346+
Then Close the pipeline logs
347+
Then Verify the pipeline status is "Succeeded"
348+
Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table

‎src/e2e-test/features/bigquery/source/BigQueryToBigQuery.feature‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,3 +262,97 @@ Feature: BigQuery source - Verification of BigQuery to BigQuery successful data
262262
Then Open and capture logs
263263
Then Verify the pipeline status is "Succeeded"
264264
Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table
265+
266+
@BQ_EXISTING_SOURCE_TEST @BQ_EXISTING_SINK_TEST @EXISTING_BQ_CONNECTION
267+
Scenario: Validate user is able to read data from BigQuery source(existing table) and store them in BigQuery sink(existing table) with use connection functionality
268+
Given Open Datafusion Project to configure pipeline
269+
When Expand Plugin group in the LHS plugins list: "Source"
270+
When Select plugin: "BigQuery" from the plugins list as: "Source"
271+
When Expand Plugin group in the LHS plugins list: "Sink"
272+
When Select plugin: "BigQuery" from the plugins list as: "Sink"
273+
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
274+
Then Navigate to the properties page of plugin: "BigQuery"
275+
Then Click plugin property: "switch-useConnection"
276+
Then Click on the Browse Connections button
277+
Then Select connection: "bqConnectionName"
278+
Then Click on the Browse button inside plugin properties
279+
Then Select connection data row with name: "dataset"
280+
Then Select connection data row with name: "bqSourceTable"
281+
Then Wait till connection data loading completes with a timeout of 60 seconds
282+
Then Verify input plugin property: "dataset" contains value: "dataset"
283+
Then Verify input plugin property: "table" contains value: "bqSourceTable"
284+
Then Click on the Get Schema button
285+
Then Validate "BigQuery" plugin properties
286+
And Close the Plugin Properties page
287+
Then Navigate to the properties page of plugin: "BigQuery2"
288+
Then Click plugin property: "useConnection"
289+
Then Click on the Browse Connections button
290+
Then Select connection: "bqConnectionName"
291+
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
292+
Then Click on the Browse button inside plugin properties
293+
Then Click SELECT button inside connection data row with name: "dataset"
294+
Then Wait till connection data loading completes with a timeout of 60 seconds
295+
Then Verify input plugin property: "dataset" contains value: "dataset"
296+
Then Enter input plugin property: "table" with value: "bqTargetTable"
297+
Then Validate "BigQuery" plugin properties
298+
Then Close the BigQuery properties
299+
Then Save the pipeline
300+
Then Preview and run the pipeline
301+
Then Wait till pipeline preview is in running state
302+
Then Open and capture pipeline preview logs
303+
Then Verify the preview run status of pipeline in the logs is "succeeded"
304+
Then Close the pipeline logs
305+
Then Close the preview
306+
Then Deploy the pipeline
307+
Then Run the Pipeline in Runtime
308+
Then Wait till pipeline is in running state
309+
Then Open and capture logs
310+
Then Verify the pipeline status is "Succeeded"
311+
Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqExpectedFile"
312+
313+
@BQ_EXISTING_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
314+
Scenario: Validate user is able to read data from BigQuery source(existing table) without clicking on the validate button of BigQuery source and store them in BigQuery sink(new table) with use connection functionality
315+
Given Open Datafusion Project to configure pipeline
316+
When Expand Plugin group in the LHS plugins list: "Source"
317+
When Select plugin: "BigQuery" from the plugins list as: "Source"
318+
When Expand Plugin group in the LHS plugins list: "Sink"
319+
When Select plugin: "BigQuery" from the plugins list as: "Sink"
320+
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
321+
Then Navigate to the properties page of plugin: "BigQuery"
322+
Then Click plugin property: "switch-useConnection"
323+
Then Click on the Browse Connections button
324+
Then Select connection: "bqConnectionName"
325+
Then Click on the Browse button inside plugin properties
326+
Then Select connection data row with name: "dataset"
327+
Then Select connection data row with name: "bqSourceTable"
328+
Then Wait till connection data loading completes with a timeout of 60 seconds
329+
Then Verify input plugin property: "dataset" contains value: "dataset"
330+
Then Verify input plugin property: "table" contains value: "bqSourceTable"
331+
Then Click on the Get Schema button
332+
Then Validate "BigQuery" plugin properties
333+
And Close the Plugin Properties page
334+
Then Navigate to the properties page of plugin: "BigQuery2"
335+
Then Click plugin property: "useConnection"
336+
Then Click on the Browse Connections button
337+
Then Select connection: "bqConnectionName"
338+
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
339+
Then Click on the Browse button inside plugin properties
340+
Then Click SELECT button inside connection data row with name: "dataset"
341+
Then Wait till connection data loading completes with a timeout of 60 seconds
342+
Then Verify input plugin property: "dataset" contains value: "dataset"
343+
Then Enter input plugin property: "table" with value: "bqTargetTable"
344+
Then Validate "BigQuery" plugin properties
345+
Then Close the BigQuery properties
346+
Then Save the pipeline
347+
Then Preview and run the pipeline
348+
Then Wait till pipeline preview is in running state
349+
Then Open and capture pipeline preview logs
350+
Then Verify the preview run status of pipeline in the logs is "succeeded"
351+
Then Close the pipeline logs
352+
Then Close the preview
353+
Then Deploy the pipeline
354+
Then Run the Pipeline in Runtime
355+
Then Wait till pipeline is in running state
356+
Then Open and capture logs
357+
Then Verify the pipeline status is "Succeeded"
358+
Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BQValidationExistingTables.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package io.cdap.plugin.bigquery.stepsdesign;
2+
3+
import com.esotericsoftware.minlog.Log;
4+
import com.google.cloud.bigquery.FieldValueList;
5+
import com.google.cloud.bigquery.TableResult;
6+
import com.google.gson.Gson;
7+
import com.google.gson.JsonElement;
8+
import com.google.gson.JsonObject;
9+
import io.cdap.e2e.utils.BigQueryClient;
10+
import io.cdap.e2e.utils.PluginPropertyUtils;
11+
import io.cucumber.core.logging.Logger;
12+
import io.cucumber.core.logging.LoggerFactory;
13+
14+
import java.io.BufferedReader;
15+
import java.io.FileReader;
16+
import java.io.IOException;
17+
import java.net.URISyntaxException;
18+
import java.nio.file.Path;
19+
import java.nio.file.Paths;
20+
import java.util.HashMap;
21+
import java.util.Map;
22+
23+
/**
24+
* BigQuery Plugin Existing Table validation.
25+
*/
26+
public class BQValidationExistingTables {
27+
28+
private static final Logger LOG = LoggerFactory.getLogger(BQValidationExistingTables.class);
29+
private static final Gson gson = new Gson();
30+
31+
/**
32+
* Validates the actual data in a BigQuery table against the expected data in a JSON file.
33+
* @param table The name of the BigQuery table to retrieve data from.
34+
* @param fileName The name of the JSON file containing the expected data.
35+
* @return True if the actual data matches the expected data, false otherwise.
36+
*/
37+
public static boolean validateActualDataToExpectedData(String table, String fileName) throws IOException,
38+
InterruptedException, URISyntaxException {
39+
Map<String, JsonObject> bigQueryMap = new HashMap<>();
40+
Map<String, JsonObject> fileMap = new HashMap<>();
41+
Path bqExpectedFilePath = Paths.get(BQValidationExistingTables.class.getResource("/" + fileName).toURI());
42+
43+
getBigQueryTableData(table, bigQueryMap);
44+
getFileData(bqExpectedFilePath.toString(), fileMap);
45+
boolean isMatched = bigQueryMap.equals(fileMap);
46+
return isMatched;
47+
}
48+
49+
/**
50+
* Reads a JSON file line by line and populates a map with JSON objects using a specified ID key.
51+
*@param fileName The path to the JSON file to be read.
52+
* @param fileMap A map where the extracted JSON objects will be stored with their ID values as keys.
53+
*/
54+
55+
public static void getFileData(String fileName, Map<String, JsonObject> fileMap) {
56+
try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
57+
String line;
58+
while ((line = br.readLine()) != null) {
59+
JsonObject json = gson.fromJson(line, JsonObject.class);
60+
String idKey = getIdKey(json);
61+
if (idKey != null) {
62+
JsonElement idElement = json.get(idKey);
63+
if (idElement.isJsonPrimitive()) {
64+
String idValue = idElement.getAsString();
65+
fileMap.put(idValue, json);
66+
}
67+
} else {
68+
Log.error("ID key not found");
69+
}
70+
}
71+
} catch (IOException e) {
72+
Log.error("Error reading the file: " + e.getMessage());
73+
}
74+
}
75+
76+
private static void getBigQueryTableData(String targetTable, Map<String, JsonObject> bigQueryMap)
77+
throws IOException, InterruptedException {
78+
String dataset = PluginPropertyUtils.pluginProp("dataset");
79+
String projectId = PluginPropertyUtils.pluginProp("projectId");
80+
String selectQuery = "SELECT TO_JSON(t) FROM `" + projectId + "." + dataset + "." + targetTable + "` AS t";
81+
TableResult result = BigQueryClient.getQueryResult(selectQuery);
82+
83+
for (FieldValueList row : result.iterateAll()) {
84+
JsonObject json = gson.fromJson(row.get(0).getStringValue(), JsonObject.class);
85+
String idKey = getIdKey(json); // Get the actual ID key from the JSON object
86+
if (idKey != null) {
87+
JsonElement idElement = json.get(idKey);
88+
if (idElement.isJsonPrimitive()) {
89+
String id = idElement.getAsString();
90+
bigQueryMap.put(id, json);
91+
} else {
92+
Log.error("Data Mismatched");
93+
}
94+
}
95+
}
96+
}
97+
98+
/**
99+
* Retrieves the key for the ID element in the provided JSON object.
100+
*
101+
* @param json The JSON object to search for the ID key.
102+
*/
103+
private static String getIdKey(JsonObject json) {
104+
if (json.has("ID")) {
105+
return "ID";
106+
} else if (json.has("Name")) {
107+
return "Name";
108+
} else if (json.has("Price")) {
109+
return "Price";
110+
} else if (json.has("Customer_Exists")) {
111+
return "Customer_Exists";
112+
} else {
113+
return null;
114+
}
115+
}
116+
}

0 commit comments

Comments
 (0)