Add change for File Identifier, macro for Oauth 2 and access token fields and other bugs.

Add change for File Identifier, macro for Oauth 2 and access token fields.

Author: vikasrathee-cs

docs/GoogleDrive-batchsink.md

@@ -43,6 +43,10 @@ Make sure that:
 OAuth2 client credentials can be generated on Google Cloud
 [Credentials Page](https://console.cloud.google.com/apis/credentials)
 
+**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided,
+or a client id, client secret, and refresh token can be provided.
+
+**Access Token:** Short lived access token for connect.
 **Client ID:** OAuth2 client id used to identify the application.
 
 **Client Secret:** OAuth2 client secret used to access the authorization server.

docs/GoogleDrive-batchsource.md

@@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc
 ```
 Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`.
 
+**File Identifier:** Identifier of the file.
+
+This comes after `file/d/ or document/d/ or spreadsheets/d/` in the URL. For example, if the URL is
+```
+https://docs.google.com/file/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows
+```
+Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`.  
+Either Directory Identifier or File Identifier should have a value. Filters will not work 
+while providing File Identifier.
+
 **File Metadata Properties:** Properties that represent metadata of files. 
 They will be a part of output structured record. Descriptions for properties can be view at 
 [Drive API file reference](https://developers.google.com/drive/api/v3/reference/files).
@@ -58,6 +68,11 @@ Make sure that:
 OAuth2 client credentials can be generated on Google Cloud
 [Credentials Page](https://console.cloud.google.com/apis/credentials)
 
+**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided,
+or a client id, client secret, and refresh token can be provided.
+
+**Access Token:** Short lived access token for connect.
+
 **Client ID:** OAuth2 client id used to identify the application.
 
 **Client Secret:** OAuth2 client secret used to access the authorization server.

docs/GoogleSheets-batchsink.md

@@ -46,6 +46,11 @@ Make sure that:
 OAuth2 client credentials can be generated on Google Cloud
 [Credentials Page](https://console.cloud.google.com/apis/credentials)
 
+**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided,
+or a client id, client secret, and refresh token can be provided.
+
+**Access Token:** Short lived access token for connect.
+
 **Client ID:** OAuth2 client id used to identify the application.
 
 **Client Secret:** OAuth2 client secret used to access the authorization server.

docs/GoogleSheets-batchsource.md

@@ -17,6 +17,16 @@ https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourc
 ```
 Then the Directory Identifier would be `1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g`.
 
+**File Identifier:** Identifier of the spreadsheet file.
+
+This comes after `spreadsheets/d/` in the URL. For example, if the URL is
+```
+https://docs.google.com/spreadsheets/d/17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows
+```
+Then the File Identifier would be `17W3vOhBwe0i24OdVNsbz8rAMClzUitKeAbumTqWFrkows`.  
+Either Directory Identifier or File Identifier should have a value. Filters will not work
+while providing File Identifier.
+
 ### Filtering
 
 **Filter:** Filter that can be applied to the files in the selected directory. 
@@ -55,6 +65,11 @@ Make sure that:
 OAuth2 client credentials can be generated on Google Cloud 
 [Credentials Page](https://console.cloud.google.com/apis/credentials)
 
+**OAuth Method:** The method used to get OAuth access tokens. The oauth access token can be directly provided,
+or a client id, client secret, and refresh token can be provided.
+
+**Access Token:** Short lived access token for connect.
+
 **Client ID:** OAuth2 client id used to identify the application.
 
 **Client Secret:** OAuth2 client secret used to access the authorization server.
@@ -124,10 +139,10 @@ _Treat first row as column names_ - the plugin uses first row for schema definin
 **Column Names Row Number:** Number of the row to be treated as a header.
 Only shown when the 'Column Names Selection' field is set to 'Custom row as column names' header.
 
-**Number of Columns to Read:** Last column plugin will read as data. It will be ignored if the Column 
-Names Row contains less number of columns.
+**Number of Columns to Read:** Last column plugin will read as data. It will be ignored if the Column
+Names Row contains less number of columns. Set it to 0 to read all the columns in the sheet.
 
-**Number of Rows to Read:** Last row plugin will read as data.
+**Number of Rows to Read:** Last row plugin will read as data. Set it to 0 to read all the rows in the sheet.
 
 **Read Buffer Size:** Number of rows the source reads with a single API request. Default value is 100.
 

src/main/java/io/cdap/plugin/google/common/GoogleAuthBaseConfig.java

@@ -47,8 +47,14 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   public static final String CLIENT_SECRET_LABEL = "Client secret";
   public static final String REFRESH_TOKEN = "refreshToken";
   public static final String REFRESH_TOKEN_LABEL = "Refresh token";
+
+  public static final String ACCESS_TOKEN = "accessToken";
+  public static final String ACCESS_TOKEN_LABEL = "Access token";
+
+  public static final String OAUTH_METHOD = "oauthMethod";
   public static final String ACCOUNT_FILE_PATH = "accountFilePath";
   public static final String DIRECTORY_IDENTIFIER = "directoryIdentifier";
+  public static final String FILE_IDENTIFIER = "fileIdentifier";
   public static final String NAME_SERVICE_ACCOUNT_TYPE = "serviceAccountType";
   public static final String NAME_SERVICE_ACCOUNT_JSON = "serviceAccountJSON";
   public static final String SERVICE_ACCOUNT_FILE_PATH = "filePath";
@@ -76,21 +82,38 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   @Macro
   protected String serviceAccountType;
 
+  @Macro
   @Nullable
+  @Name(OAUTH_METHOD)
+  @Description("The method used to get OAuth access tokens. "
+    + "The oauth access token can be directly provided, "
+    + "or a client id, client secret, and refresh token can be provided.")
+  private String oauthMethod;
+
+  @Nullable
+  @Macro
   @Name(CLIENT_ID)
   @Description("OAuth2 client id.")
   private String clientId;
 
   @Nullable
+  @Macro
   @Name(CLIENT_SECRET)
   @Description("OAuth2 client secret.")
   private String clientSecret;
 
   @Nullable
+  @Macro
   @Name(REFRESH_TOKEN)
   @Description("OAuth2 refresh token.")
   private String refreshToken;
 
+  @Nullable
+  @Macro
+  @Name(ACCESS_TOKEN)
+  @Description("Short lived access token for connect.")
+  private String accessToken;
+
   @Nullable
   @Macro
   @Name(ACCOUNT_FILE_PATH)
@@ -108,12 +131,22 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
   @Macro
   protected String serviceAccountJson;
 
+  @Nullable
+  @Macro
   @Name(DIRECTORY_IDENTIFIER)
   @Description("Identifier of the folder. This comes after “folders/” in the URL. For example, if the URL was " +
     "“https://drive.google.com/drive/folders/1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g?resourcekey=0-XVijrJSp3E3gkdJp20MpCQ”, "
     + "then the Directory Identifier would be “1dyUEebJaFnWa3Z4n0BFMVAXQ7mfUH11g”.")
   private String directoryIdentifier;
 
+  @Nullable
+  @Macro
+  @Name(FILE_IDENTIFIER)
+  @Description("Identifier of the file. This comes after “file/d/ or spreadsheets/d/ or document/d/” in the URL. " +
+    "For example, if the URL was “https://drive.google.com/file/d/16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1/view”, "
+    + "then the File Identifier would be “16npTpL3ozkAzB5kLQ-oQD3IlTZhnnh2w1”.")
+  private String fileIdentifier;
+
   /**
    * Returns the ValidationResult.
    *
@@ -122,7 +155,7 @@ public abstract class GoogleAuthBaseConfig extends PluginConfig {
    */
   public ValidationResult validate(FailureCollector collector) {
     IdUtils.validateReferenceName(referenceName, collector);
-
+    checkIfDirectoryOrFileIdentifierExists(collector);
     ValidationResult validationResult = new ValidationResult();
     if (validateAuthType(collector)) {
       AuthType authType = getAuthType();
@@ -143,13 +176,10 @@ public ValidationResult validate(FailureCollector collector) {
         try {
           GoogleDriveClient client = new GoogleDriveClient(this);
 
-          // validate auth
-          validateCredentials(collector, client);
-
-          // validate directory
-          validateDirectoryIdentifier(collector, client);
-
-          validationResult.setDirectoryAccessible(true);
+          // check directory or file access
+          if (isDirectoryOrFileAccessible(collector, client)) {
+            validationResult.setDirectoryOrFileAccessible(true);
+          }
         } catch (Exception e) {
           collector.addFailure(
             String.format("Exception during authentication/directory properties check: %s.", e.getMessage()),
@@ -177,9 +207,13 @@ private boolean validateAuthType(FailureCollector collector) {
   }
 
   private boolean validateOAuth2Properties(FailureCollector collector) {
-    return checkPropertyIsSet(collector, clientId, CLIENT_ID, CLIENT_ID_LABEL)
-      & checkPropertyIsSet(collector, clientSecret, CLIENT_SECRET, CLIENT_SECRET_LABEL)
-      & checkPropertyIsSet(collector, refreshToken, REFRESH_TOKEN, REFRESH_TOKEN_LABEL);
+    if (OAuthMethod.REFRESH_TOKEN.equals(getOAuthMethod())) {
+      return checkPropertyIsSet(collector, clientId, CLIENT_ID, CLIENT_ID_LABEL)
+        & checkPropertyIsSet(collector, clientSecret, CLIENT_SECRET, CLIENT_SECRET_LABEL)
+        & checkPropertyIsSet(collector, refreshToken, REFRESH_TOKEN, REFRESH_TOKEN_LABEL);
+    } else {
+      return checkPropertyIsSet(collector, accessToken, ACCESS_TOKEN, ACCESS_TOKEN_LABEL);
+    }
   }
 
   private boolean validateServiceAccount(FailureCollector collector) {
@@ -206,27 +240,41 @@ private boolean validateServiceAccount(FailureCollector collector) {
     return collector.getValidationFailures().size() == 0;
   }
 
-  private void validateCredentials(FailureCollector collector, GoogleDriveClient driveClient) throws IOException {
-    try {
-      driveClient.checkRootFolder();
-    } catch (GoogleJsonResponseException e) {
-      collector.addFailure(e.getDetails().getMessage(), "Provide valid credentials.")
-        .withConfigProperty(NAME_SERVICE_ACCOUNT_TYPE)
-        .withStacktrace(e.getStackTrace());
-    }
-  }
-
-  private void validateDirectoryIdentifier(FailureCollector collector, GoogleDriveClient driveClient)
+  private boolean isDirectoryOrFileAccessible(FailureCollector collector, GoogleDriveClient driveClient)
     throws IOException {
-    if (!containsMacro(DIRECTORY_IDENTIFIER)) {
+
+    if (directoryIdentifier != null && !containsMacro(DIRECTORY_IDENTIFIER)) {
       try {
         driveClient.isFolderAccessible(directoryIdentifier);
+        return true;
       } catch (GoogleJsonResponseException e) {
         collector.addFailure(e.getDetails().getMessage(), "Provide an existing folder identifier.")
           .withConfigProperty(DIRECTORY_IDENTIFIER)
           .withStacktrace(e.getStackTrace());
       }
     }
+
+    if (fileIdentifier != null && !containsMacro(FILE_IDENTIFIER)) {
+      try {
+        driveClient.isFileAccessible(fileIdentifier);
+        return true;
+      } catch (GoogleJsonResponseException e) {
+        collector.addFailure(e.getDetails().getMessage(), "Provide an existing file identifier.")
+          .withConfigProperty(FILE_IDENTIFIER)
+          .withStacktrace(e.getStackTrace());
+      }
+    }
+    return false;
+  }
+
+  protected void checkIfDirectoryOrFileIdentifierExists(FailureCollector collector) {
+    if (directoryIdentifier == null && !containsMacro(DIRECTORY_IDENTIFIER) &&
+      fileIdentifier == null && !containsMacro(FILE_IDENTIFIER)) {
+      collector.addFailure("Both Directory Identifier and File Identifier can not be null.",
+                           "Provide either Directory Identifier or File Identifier.")
+        .withConfigProperty(DIRECTORY_IDENTIFIER)
+        .withConfigProperty(FILE_IDENTIFIER);
+    }
   }
 
   protected boolean checkPropertyIsSet(FailureCollector collector, String propertyValue, String propertyName,
@@ -250,6 +298,10 @@ public String getDirectoryIdentifier() {
     return directoryIdentifier;
   }
 
+  public String getFileIdentifier() {
+    return fileIdentifier;
+  }
+
   public AuthType getAuthType() {
     return AuthType.fromValue(authType);
   }
@@ -277,6 +329,13 @@ public void setAccountFilePath(String accountFilePath) {
   public void setDirectoryIdentifier(String directoryIdentifier) {
     this.directoryIdentifier = directoryIdentifier;
   }
+  public void setFileIdentifier(String fileIdentifier) {
+    this.fileIdentifier = fileIdentifier;
+  }
+
+  public void setOauthMethod(String oauthMethod) {
+    this.oauthMethod = oauthMethod;
+  }
 
   public void setClientId(String clientId) {
     this.clientId = clientId;
@@ -290,6 +349,10 @@ public void setRefreshToken(String refreshToken) {
     this.refreshToken = refreshToken;
   }
 
+  public void setAccessToken(String accessToken) {
+    this.accessToken = accessToken;
+  }
+
   @Nullable
   public String getClientId() {
     return clientId;
@@ -305,6 +368,11 @@ public String getRefreshToken() {
     return refreshToken;
   }
 
+  @Nullable
+  public String getAccessToken() {
+    return accessToken;
+  }
+
   @Nullable
   public String getServiceAccountFilePath() {
     if (containsMacro(ACCOUNT_FILE_PATH) || Strings.isNullOrEmpty(accountFilePath)
@@ -344,4 +412,15 @@ public Boolean isServiceAccountFilePath() {
     String serviceAccountType = getServiceAccountType();
     return Strings.isNullOrEmpty(serviceAccountType) ? null : serviceAccountType.equals(SERVICE_ACCOUNT_FILE_PATH);
   }
+
+  public OAuthMethod getOAuthMethod() {
+    if (oauthMethod == null) {
+      return OAuthMethod.REFRESH_TOKEN;
+    }
+    try {
+      return OAuthMethod.valueOf(oauthMethod.toUpperCase());
+    } catch (IllegalArgumentException e) {
+      throw new IllegalArgumentException("Invalid oauth method " + oauthMethod);
+    }
+  }
 }

src/main/java/io/cdap/plugin/google/common/GoogleDriveClient.java

@@ -43,7 +43,6 @@
  */
 public class GoogleDriveClient<C extends GoogleAuthBaseConfig> {
   private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
-  private static final String ROOT_FOLDER_ID = "root";
   protected final Drive service;
   protected final C config;
   protected NetHttpTransport httpTransport;
@@ -88,13 +87,18 @@ protected Drive getDriveClient() throws IOException {
   }
 
   protected Credential getOAuth2Credential() {
-    GoogleCredential credential = new GoogleCredential.Builder()
-        .setTransport(httpTransport)
-        .setJsonFactory(JSON_FACTORY)
-        .setClientSecrets(config.getClientId(),
-            config.getClientSecret())
-        .build();
-    credential.createScoped(getRequiredScopes()).setRefreshToken(config.getRefreshToken());
+    GoogleCredential credential;
+    GoogleCredential.Builder builder = new GoogleCredential.Builder()
+      .setTransport(httpTransport)
+      .setJsonFactory(JSON_FACTORY);
+    if (OAuthMethod.ACCESS_TOKEN.equals(config.getOAuthMethod())) {
+      credential = builder.build();
+      credential.createScoped(getRequiredScopes()).setAccessToken(config.getAccessToken());
+    } else {
+      credential = builder.setClientSecrets(config.getClientId(),
+                                            config.getClientSecret()).build();
+      credential.createScoped(getRequiredScopes()).setRefreshToken(config.getRefreshToken());
+    }
     return credential;
   }
 
@@ -121,11 +125,11 @@ protected List<String> getRequiredScopes() {
     return Collections.singletonList(DriveScopes.DRIVE_READONLY);
   }
 
-  public void checkRootFolder() throws IOException {
-    service.files().get(ROOT_FOLDER_ID).setSupportsAllDrives(true).execute();
-  }
-
   public void isFolderAccessible(String folderId) throws IOException {
     service.files().get(folderId).setSupportsAllDrives(true).execute();
   }
+
+  public void isFileAccessible(String fileId) throws IOException {
+    service.files().get(fileId).setSupportsAllDrives(true).execute();
+  }
 }

src/main/java/io/cdap/plugin/google/common/GoogleDriveFilteringClient.java

@@ -71,6 +71,10 @@ public List<File> getFilesSummary(List<ExportedType> exportedTypes, int filesNum
       String nextToken = "";
       int retrievedFiles = 0;
       int actualFilesNumber = filesNumber;
+      if (config.getFileIdentifier() != null) {
+        files.add(service.files().get(config.getFileIdentifier()).setSupportsAllDrives(true).execute());
+        return files;
+      }
       Drive.Files.List request = service.files().list()
         .setSupportsAllDrives(true)
         .setIncludeItemsFromAllDrives(true)