app: load revel and alphamissense in the collection missense_variation_functional_score, #TASK-8163

Author: jtarraga

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/admin/executors/LoadCommandExecutor.java

@@ -22,6 +22,7 @@
 import org.opencb.cellbase.app.cli.admin.AdminCliOptionsParser;
 import org.opencb.cellbase.core.config.SpeciesConfiguration;
 import org.opencb.cellbase.core.exception.CellBaseException;
+import org.opencb.cellbase.core.models.DataSource;
 import org.opencb.cellbase.core.models.Release;
 import org.opencb.cellbase.core.result.CellBaseDataResult;
 import org.opencb.cellbase.core.utils.DatabaseNameUtils;
@@ -359,17 +360,25 @@ private void loadProteinFunctionalPrediction() throws NoSuchMethodException, Int
     }
 
     private void loadRevel() throws CellBaseException {
+        // Check if REVEL source has already been loaded
+        checkSourceAlreadyLoaded(REVEL_DATA);
+
         HashMap<String, String> collectionMap = new HashMap<>();
-        collectionMap.put(PROTEIN_SUBSTITUTION_PREDICTION_DATA, REVEL_DATA + JSON_GZ_EXTENSION);
+        collectionMap.put(MISSENSE_VARIATION_SCORE_DATA, REVEL_DATA + JSON_GZ_EXTENSION);
 
-        loadData(input.resolve(PROTEIN_SUBSTITUTION_PREDICTION_DATA).resolve(REVEL_DATA), collectionMap);
+        Path revelPath = input.resolve(PROTEIN_SUBSTITUTION_PREDICTION_DATA).resolve(REVEL_DATA);
+        loadData(revelPath, collectionMap);
     }
 
     private void loadAlphaMissense() throws CellBaseException {
+        // Check if AlphaMissense source has already been loaded
+        checkSourceAlreadyLoaded(ALPHAMISSENSE_DATA);
+
         HashMap<String, String> collectionMap = new HashMap<>();
-        collectionMap.put(PROTEIN_SUBSTITUTION_PREDICTION_DATA, ALPHAMISSENSE_DATA + JSON_GZ_EXTENSION);
+        collectionMap.put(MISSENSE_VARIATION_SCORE_DATA, ALPHAMISSENSE_DATA + JSON_GZ_EXTENSION);
 
-        loadData(input.resolve(PROTEIN_SUBSTITUTION_PREDICTION_DATA).resolve(ALPHAMISSENSE_DATA), collectionMap);
+        Path alphaMissensePath = input.resolve(PROTEIN_SUBSTITUTION_PREDICTION_DATA).resolve(ALPHAMISSENSE_DATA);
+        loadData(alphaMissensePath, collectionMap);
     }
 
     private void loadClinical() throws FileNotFoundException {
@@ -677,4 +686,16 @@ private Release getDataReleaseForLoading(DataReleaseManager dataReleaseManager)
         }
         return lastDataRelease;
     }
+
+    private void checkSourceAlreadyLoaded(String sourceId) throws CellBaseException {
+        Release release = getDataReleaseForLoading(dataReleaseManager);
+        if (release.getSources() != null) {
+            for (DataSource source : release.getSources()) {
+                if (sourceId.equalsIgnoreCase(source.getId())) {
+                    throw new CellBaseException("Loading data '" + sourceId + "' with release " + dataRelease
+                            + " failed: source '" + sourceId + "' already loaded previously");
+                }
+            }
+        }
+    }
 }

cellbase-lib/src/main/java/org/opencb/cellbase/lib/loader/MongoDBCellBaseLoader.java

@@ -51,6 +51,8 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.stream.Collectors;
 
+import static org.opencb.cellbase.lib.EtlCommons.MISSENSE_VARIATION_SCORE_DATA;
+
 /**
  * Created by parce on 18/02/15.
  */
@@ -137,13 +139,19 @@ private String getCollectionName() throws LoaderException {
             throw new LoaderException("Invalid data release " + dataRelease + " for database " + database + ". Available releases"
                     + " are: " + StringUtils.join(releases, ","));
         }
-        for (Release dr : result.getResults()) {
-            if (dr.getRelease() == dataRelease) {
-                if (dr.getCollections().containsKey(data) && dr.getCollections().get(data).equals(collectionName)) {
-                    throw new LoaderException("Loading new data " + data + " with release " + dataRelease
-                            + " (already populated previously)");
+
+        // Sanity check don't populate collections already populated
+        // Missense variation score data (i.e., revel and alphaMissense) is checked later, since revel and alphamissense are loaded
+        // in the same collection but independently
+        if (!data.equalsIgnoreCase(MISSENSE_VARIATION_SCORE_DATA)) {
+            for (Release dr : result.getResults()) {
+                if (dr.getRelease() == dataRelease) {
+                    if (dr.getCollections().containsKey(data) && dr.getCollections().get(data).equals(collectionName)) {
+                        throw new LoaderException("Loading new data '" + data + "' with release " + dataRelease
+                                + " (already populated previously)");
+                    }
+                    break;
                 }
-                break;
             }
         }