Add change for File Identifier, macro for Oauth 2 fields.

Add change for File Identifier, macro for Oauth 2 fields.

Author: vikasrathee-cs

docs/GoogleDrive-batchsource.md

@@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc
 ```
 Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`.
 
+**File Identifier:** Identifier of the file.
+
+This comes after `file/d/ or document/d/ or spreadsheets/d/` in the URL. For example, if the URL is
+```
+https://docs.google.com/file/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows
+```
+Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`.  
+Either Directory Identifier or File Identifier should have a value. Filters will not work 
+while providing File Identifier.
+
 **File Metadata Properties:** Properties that represent metadata of files. 
 They will be a part of output structured record. Descriptions for properties can be view at 
 [Drive API file reference](https://developers.google.com/drive/api/v3/reference/files).

docs/GoogleSheets-batchsource.md

@@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc
 ```
 Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`.
 
+**File Identifier:** Identifier of the spreadsheet file.
+
+This comes after `spreadsheets/d/` in the URL. For example, if the URL is
+```
+https://docs.google.com/spreadsheets/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows
+```
+Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`.  
+Either Directory Identifier or File Identifier should have a value. Filters will not work
+while providing File Identifier.
+
 ### Filtering
 
 **Filter:** Filter that can be applied to the files in the selected directory. 

src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java

@@ -49,6 +49,7 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   public static final String REFRESH_TOKEN_LABEL = "Refresh token";
   public static final String ACCOUNT_FILE_PATH = "accountFilePath";
   public static final String DIRECTORY_IDENTIFIER = "directoryIdentifier";
+  public static final String FILE_IDENTIFIER = "fileIdentifier";
   public static final String NAME_SERVICE_ACCOUNT_TYPE = "serviceAccountType";
   public static final String NAME_SERVICE_ACCOUNT_JSON = "serviceAccountJSON";
   public static final String SERVICE_ACCOUNT_FILE_PATH = "filePath";
@@ -77,16 +78,19 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   protected String serviceAccountType;
 
   @Nullable
+  @Macro
   @Name(CLIENT_ID)
   @Description("OAuth2 client id.")
   private String clientId;
 
   @Nullable
+  @Macro
   @Name(CLIENT_SECRET)
   @Description("OAuth2 client secret.")
   private String clientSecret;
 
   @Nullable
+  @Macro
   @Name(REFRESH_TOKEN)
   @Description("OAuth2 refresh token.")
   private String refreshToken;
@@ -108,12 +112,22 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   @Macro
   protected String serviceAccountJson;
 
+  @Nullable
+  @Macro
   @Name(DIRECTORY_IDENTIFIER)
   @Description("Identifier of the folder. This comes after “folders/” in the URL. For example, if the URL was " +
     "“https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourcekey=0-XVijrJSp3E3gkdJp20MpCQ”, "
     + "then the Directory Identifier would be “1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g”.")
   private String directoryIdentifier;
 
+  @Nullable
+  @Macro
+  @Name(FILE_IDENTIFIER)
+  @Description("Identifier of the file. This comes after “file/d/ or spreadsheets/d/ or document/d/” in the URL. " +
+    "For example, if the URL was “https://drive.google.com/file/d/16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1/view”, "
+    + "then the File Identifier would be “16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1”.")
+  private String fileIdentifier;
+
   /**
    * Returns the ValidationResult.
    *
@@ -122,7 +136,7 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
    */
   public ValidationResult validate(FailureCollector collector) {
     IdUtils.validateReferenceName(referenceName, collector);
-
+    checkIfDirectoryOrFileIdentifierExists(collector);
     ValidationResult validationResult = new ValidationResult();
     if (validateAuthType(collector)) {
       AuthType authType = getAuthType();
@@ -147,9 +161,10 @@ public ValidationResult validate(FailureCollector collector) {
           validateCredentials(collector, client);
 
           // validate directory
-          validateDirectoryIdentifier(collector, client);
-
-          validationResult.setDirectoryAccessible(true);
+          validateDirectoryAndFileIdentifier(collector, client);
+          if (!(containsMacro(FILE_IDENTIFIER) || containsMacro(DIRECTORY_IDENTIFIER))) {
+            validationResult.setDirectoryOrFileAccessible(true);
+          }
         } catch (Exception e) {
           collector.addFailure(
             String.format("Exception during authentication/directory properties check: %s.", e.getMessage()),
@@ -177,6 +192,9 @@ private boolean validateAuthType(FailureCollector collector) {
   }
 
   private boolean validateOAuth2Properties(FailureCollector collector) {
+    if (containsMacro(CLIENT_ID) || containsMacro(CLIENT_SECRET) || containsMacro(REFRESH_TOKEN)) {
+      return false;
+    }
     return checkPropertyIsSet(collector, clientId, CLIENT_ID, CLIENT_ID_LABEL)
       & checkPropertyIsSet(collector, clientSecret, CLIENT_SECRET, CLIENT_SECRET_LABEL)
       & checkPropertyIsSet(collector, refreshToken, REFRESH_TOKEN, REFRESH_TOKEN_LABEL);
@@ -222,9 +240,10 @@ private void validateCredentials(FailureCollector collector, GoogleDriveClient d
     }
   }
 
-  private void validateDirectoryIdentifier(FailureCollector collector, GoogleDriveClient driveClient)
+  private void validateDirectoryAndFileIdentifier(FailureCollector collector, GoogleDriveClient driveClient)
     throws IOException {
-    if (!containsMacro(DIRECTORY_IDENTIFIER)) {
+
+    if (directoryIdentifier != null && !containsMacro(DIRECTORY_IDENTIFIER)) {
       try {
         driveClient.isFolderAccessible(directoryIdentifier);
       } catch (GoogleJsonResponseException e) {
@@ -233,6 +252,26 @@ private void validateDirectoryIdentifier(FailureCollector collector, GoogleDrive
           .withStacktrace(e.getStackTrace());
       }
     }
+
+    if (fileIdentifier != null && !containsMacro(FILE_IDENTIFIER)) {
+      try {
+        driveClient.isFileAccessible(fileIdentifier);
+      } catch (GoogleJsonResponseException e) {
+        collector.addFailure(e.getDetails().getMessage(), "Provide an existing file identifier.")
+          .withConfigProperty(FILE_IDENTIFIER)
+          .withStacktrace(e.getStackTrace());
+      }
+    }
+  }
+
+  protected void checkIfDirectoryOrFileIdentifierExists(FailureCollector collector) {
+    if (directoryIdentifier == null && !containsMacro(DIRECTORY_IDENTIFIER) &&
+      fileIdentifier == null && !containsMacro(FILE_IDENTIFIER)) {
+      collector.addFailure("Both Directory Identifier and File Identifier can not be null.",
+                           "Provide either Directory Identifier or File Identifier.")
+        .withConfigProperty(DIRECTORY_IDENTIFIER)
+        .withConfigProperty(FILE_IDENTIFIER);
+    }
   }
 
   protected boolean checkPropertyIsSet(FailureCollector collector, String propertyValue, String propertyName,
@@ -256,6 +295,10 @@ public String getDirectoryIdentifier() {
     return directoryIdentifier;
   }
 
+  public String getFileIdentifier() {
+    return fileIdentifier;
+  }
+
   public AuthType getAuthType() {
     return AuthType.fromValue(authType);
   }
@@ -283,6 +326,9 @@ public void setAccountFilePath(String accountFilePath) {
   public void setDirectoryIdentifier(String directoryIdentifier) {
     this.directoryIdentifier = directoryIdentifier;
   }
+  public void setFileIdentifier(String fileIdentifier) {
+    this.fileIdentifier = fileIdentifier;
+  }
 
   public void setClientId(String clientId) {
     this.clientId = clientId;

src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java

@@ -113,4 +113,8 @@ public void checkRootFolder() throws IOException {
   public void isFolderAccessible(String folderId) throws IOException {
     service.files().get(folderId).setSupportsAllDrives(true).execute();
   }
+
+  public void isFileAccessible(String fileId) throws IOException {
+    service.files().get(fileId).setSupportsAllDrives(true).execute();
+  }
 }

src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java

@@ -71,6 +71,10 @@ public List<File> getFilesSummary(List<ExportedType> exportedTypes, int filesNum
       String nextToken = "";
       int retrievedFiles = 0;
       int actualFilesNumber = filesNumber;
+      if (config.getFileIdentifier() != null) {
+        files.add(service.files().get(config.getFileIdentifier()).setSupportsAllDrives(true).execute());
+        return files;
+      }
       Drive.Files.List request = service.files().list()
         .setSupportsAllDrives(true)
         .setIncludeItemsFromAllDrives(true)

src/main/java/io/cdap/plugin/google/common/ValidationResult.java

@@ -20,13 +20,13 @@
  * Wrapper to save validation results. Is used to transfer validation results.
  */
 public class ValidationResult {
-  private boolean directoryAccessible = false;
+  private boolean directoryOrFileAccessible = false;
 
-  public boolean isDirectoryAccessible() {
-    return directoryAccessible;
+  public boolean isDirectoryOrFileAccessible() {
+    return directoryOrFileAccessible;
   }
 
-  public void setDirectoryAccessible(boolean directoryAccessible) {
-    this.directoryAccessible = directoryAccessible;
+  public void setDirectoryOrFileAccessible(boolean directoryOrFileAccessible) {
+    this.directoryOrFileAccessible = directoryOrFileAccessible;
   }
 }

src/main/java/io/cdap/plugin/google/drive/source/GoogleDriveSourceConfig.java

@@ -324,6 +324,10 @@ public static GoogleDriveSourceConfig of(JsonObject properties) throws IOExcepti
       googleDriveSourceConfig.setDirectoryIdentifier(
         properties.get(GoogleDriveSourceConfig.DIRECTORY_IDENTIFIER).getAsString());
     }
+    if (properties.has(GoogleDriveSourceConfig.FILE_IDENTIFIER)) {
+      googleDriveSourceConfig.setFileIdentifier(
+        properties.get(GoogleDriveSourceConfig.FILE_IDENTIFIER).getAsString());
+    }
     if (properties.has(GoogleDriveSourceConfig.FILTER)) {
       googleDriveSourceConfig.setFilter(properties.get(GoogleDriveSourceConfig.FILTER).getAsString());
     }

src/main/java/io/cdap/plugin/google/sheets/source/GoogleSheetsSourceConfig.java

@@ -117,7 +117,7 @@ public class GoogleSheetsSourceConfig extends GoogleFilteringSourceConfig {
   @Name(NAME_SCHEMA)
   @Description("The schema of the table to read.")
   @Macro
-  private transient Schema schema = null;
+  private String schema;
 
   @Name(FORMATTING)
   @Description("Output format for numeric sheet cells. " +
@@ -267,22 +267,26 @@ public GoogleSheetsSourceConfig(String referenceName) {
    * @param collector throws validation exception
    */
   public Schema getSchema(FailureCollector collector) {
-    if (schema == null && shouldGetSchema()) {
+    Schema existingSchema = this.getSchema();
+    if (existingSchema == null && shouldGetSchema()) {
       if (dataSchemaInfo.isEmpty()) {
         collector.addFailure("There are no headers to process.",
                              "Perhaps no validation step was executed before schema generation.")
           .withConfigProperty(SCHEMA);
       }
-      schema = SchemaBuilder.buildSchema(this, new ArrayList<>(dataSchemaInfo.values()));
+      existingSchema = SchemaBuilder.buildSchema(this, new ArrayList<>(dataSchemaInfo.values()));
     }
-    return schema;
+    return existingSchema;
   }
 
   private boolean shouldGetSchema() {
     return !containsMacro(SHEETS_TO_PULL) && !containsMacro(SHEETS_IDENTIFIERS) &&
       !containsMacro(COLUMN_NAMES_SELECTION) && !containsMacro(CUSTOM_COLUMN_NAMES_ROW) &&
       !containsMacro(LAST_DATA_COLUMN) && !containsMacro(NAME_SERVICE_ACCOUNT_TYPE) &&
-      !containsMacro(ACCOUNT_FILE_PATH) && !containsMacro(NAME_SERVICE_ACCOUNT_JSON);
+      !containsMacro(ACCOUNT_FILE_PATH) && !containsMacro(NAME_SERVICE_ACCOUNT_JSON) &&
+      !containsMacro(CLIENT_ID) && !containsMacro(CLIENT_SECRET) &&
+      !containsMacro(REFRESH_TOKEN) && !containsMacro(FILE_IDENTIFIER) &&
+      !containsMacro(DIRECTORY_IDENTIFIER);
   }
 
   /**
@@ -300,7 +304,8 @@ public ValidationResult validate(FailureCollector collector) {
     validateLastDataColumnIndexAndLastRowIndex(collector);
     validateSpreadsheetAndSheetFieldNames(collector);
 
-    if (collector.getValidationFailures().isEmpty() && validationResult.isDirectoryAccessible()) {
+    if (collector.getValidationFailures().isEmpty() &&
+      (validationResult.isDirectoryOrFileAccessible())) {
       GoogleDriveFilteringClient driveClient;
       GoogleSheetsSourceClient sheetsSourceClient;
       try {
@@ -315,7 +320,7 @@ public ValidationResult validate(FailureCollector collector) {
         spreadsheetsFiles = driveClient
           .getFilesSummary(Collections.singletonList(ExportedType.SPREADSHEETS), 1);
       } catch (ExecutionException | RetryException e) {
-        collector.addFailure("Invalid search query, see https://developers.google.com/drive/api/v3/ref-search-terms",
+        collector.addFailure(String.format("Failed while getting file schema due to reason : %s", e.getMessage()),
                         null).withStacktrace(e.getStackTrace());
         return validationResult;
       }
@@ -1010,7 +1015,7 @@ public void setSheetsIdentifiers(String sheetsIdentifiers) {
   }
 
   public void setSchema(String schema) throws IOException {
-    this.schema = Schema.parseJson(schema);
+    this.schema = schema;
   }
 
   public void setFormatting(String formatting) {
@@ -1257,7 +1262,19 @@ public static GoogleSheetsSourceConfig of(JsonObject properties) throws IOExcept
       googleSheetsSourceConfig.setDirectoryIdentifier(
         properties.get(GoogleSheetsSourceConfig.DIRECTORY_IDENTIFIER).getAsString());
     }
-
+    if (properties.has(GoogleSheetsSourceConfig.FILE_IDENTIFIER)) {
+      googleSheetsSourceConfig.setFileIdentifier(
+        properties.get(GoogleSheetsSourceConfig.FILE_IDENTIFIER).getAsString());
+    }
     return googleSheetsSourceConfig;
   }
+  @Nullable
+  public Schema getSchema() {
+    try {
+      return !this.containsMacro("schema") &&
+        !Strings.isNullOrEmpty(this.schema) ? Schema.parseJson(this.schema) : null;
+    } catch (Exception var2) {
+      throw new IllegalArgumentException("Invalid schema: " + var2.getMessage(), var2);
+    }
+  }
 }

widgets/GoogleDrive-batchsource.json

@@ -17,6 +17,11 @@
           "label": "Directory Identifier",
           "name": "directoryIdentifier"
         },
+        {
+          "widget-type": "textbox",
+          "label": "File Identifier",
+          "name": "fileIdentifier"
+        },
         {
           "name": "fileMetadataProperties",
           "widget-type": "multi-select",

widgets/GoogleSheets-batchsource.json

@@ -16,6 +16,11 @@
           "widget-type": "textbox",
           "label": "Directory Identifier",
           "name": "directoryIdentifier"
+        },
+        {
+          "widget-type": "textbox",
+          "label": "File Identifier",
+          "name": "fileIdentifier"
         }
       ]
     },
@@ -366,7 +371,12 @@
       ]
     }
   ],
-  "outputs": [],
+  "outputs": [
+    {
+      "name": "schema",
+      "widget-type": "schema"
+    }
+  ],
   "filters": [
     {
       "name": "Select modification date range",