Merge pull request #634 from ibi-group/feature/DT-530-additional-feed-source-summary-fields

Additional feed summary params

Author: br648

src/main/java/com/conveyal/datatools/manager/controllers/DumpController.java

@@ -1,9 +1,9 @@
 package com.conveyal.datatools.manager.controllers;
 
-import com.conveyal.datatools.common.status.MonitorableJob;
 import com.conveyal.datatools.manager.auth.Auth0UserProfile;
 import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob;
 import com.conveyal.datatools.manager.jobs.ValidateFeedJob;
+import com.conveyal.datatools.manager.jobs.ValidateGtfsPlusFeedJob;
 import com.conveyal.datatools.manager.jobs.ValidateMobilityDataFeedJob;
 import com.conveyal.datatools.manager.models.Deployment;
 import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty;
@@ -365,6 +365,7 @@ public static boolean validateAll (boolean load, boolean force, String filterFee
             } else {
                 JobUtils.heavyExecutor.execute(new ValidateFeedJob(version, systemUser, false));
                 JobUtils.heavyExecutor.execute(new ValidateMobilityDataFeedJob(version, systemUser, false));
+                JobUtils.heavyExecutor.execute(new ValidateGtfsPlusFeedJob(version, systemUser, false));
             }
         }
         // ValidateAllFeedsJob validateAllFeedsJob = new ValidateAllFeedsJob("system", force, load);

src/main/java/com/conveyal/datatools/manager/controllers/api/GtfsPlusController.java

@@ -249,17 +249,17 @@ private static String publishGtfsPlusFile(Request req, Response res) {
     }
 
     /**
-     * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+).
+     * HTTP endpoint that validates GTFS+ tables for a specific feed version (or its saved/edited GTFS+). If the feed
+     * version already has GTFS+ validation results, those will be returned instead of re-validating.
      */
     private static GtfsPlusValidation getGtfsPlusValidation(Request req, Response res) {
-        String feedVersionId = req.params("versionid");
-        GtfsPlusValidation gtfsPlusValidation = null;
         try {
-            gtfsPlusValidation = GtfsPlusValidation.validate(feedVersionId);
+            String feedVersionId = req.params("versionid");
+            return GtfsPlusValidation.validate(feedVersionId);
         } catch(Exception e) {
             logMessageAndHalt(req, 500, "Could not read GTFS+ zip file", e);
         }
-        return gtfsPlusValidation;
+        return null;
     }
 
     /**

src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java

@@ -49,22 +49,47 @@ private GtfsPlusValidation (String feedVersionId) {
         this.feedVersionId = feedVersionId;
     }
 
+    public GtfsPlusValidation(String feedVersionId, boolean published, List<ValidationIssue> issues) {
+        this(feedVersionId);
+        this.published = published;
+        this.issues = issues;
+    }
+
+    public GtfsPlusValidation() {
+        this(null);
+        // Constructor for serialization.
+    }
+
     /**
-     * Validate a GTFS+ feed and return a list of issues encountered.
-     * FIXME: For now this uses the MapDB-backed GTFSFeed class. Which actually suggests that this might
-     *   should be contained within a MonitorableJob.
+     * Overload method to retrieve the feed version.
      */
     public static GtfsPlusValidation validate(String feedVersionId) throws Exception {
-        GtfsPlusValidation validation = new GtfsPlusValidation(feedVersionId);
+        FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId);
+        if (feedVersion != null) {
+            if (feedVersion.gtfsPlusValidation != null) {
+                return feedVersion.gtfsPlusValidation;
+            }
+            GtfsPlusValidation gtfsPlusValidation = validate(feedVersion);
+            feedVersion.gtfsPlusValidation = gtfsPlusValidation;
+            Persistence.feedVersions.replace(feedVersion.id, feedVersion);
+            return gtfsPlusValidation;
+        }
+        return null;
+    }
+
+    /**
+     * Validate a GTFS+ feed and return a list of issues encountered.
+     */
+    public static GtfsPlusValidation validate(FeedVersion feedVersion) throws Exception {
         if (!DataManager.isModuleEnabled("gtfsplus")) {
             throw new IllegalStateException("GTFS+ module must be enabled in server.yml to run GTFS+ validation.");
         }
-        LOG.info("Validating GTFS+ for {}", feedVersionId);
+        GtfsPlusValidation validation = new GtfsPlusValidation(feedVersion.id);
+        LOG.info("Validating GTFS+ for {}", feedVersion.id);
 
-        FeedVersion feedVersion = Persistence.feedVersions.getById(feedVersionId);
         // Load the main GTFS file.
         // FIXME: Swap MapDB-backed GTFSFeed for use of SQL data?
-        File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersionId + ".db");
+        File gtfsFeedDbFile = gtfsPlusStore.getFeedFile(feedVersion.id + ".db");
         String gtfsFeedDbFilePath = gtfsFeedDbFile.getAbsolutePath();
         GTFSFeed gtfsFeed;
         try {
@@ -87,7 +112,7 @@ public static GtfsPlusValidation validate(String feedVersionId) throws Exception
         }
 
         // check for saved GTFS+ data
-        File file = gtfsPlusStore.getFeed(feedVersionId);
+        File file = gtfsPlusStore.getFeed(feedVersion.id);
         if (file == null) {
             validation.published = true;
             LOG.warn("GTFS+ Validation -- Modified GTFS+ file not found, loading from main version GTFS.");

src/main/java/com/conveyal/datatools/manager/gtfsplus/ValidationIssue.java

@@ -10,6 +10,10 @@ public class ValidationIssue implements Serializable {
     public int rowIndex;
     public String description;
 
+    public ValidationIssue() {
+        // Empty constructor for serialization
+    }
+
     public ValidationIssue(String tableId, String fieldName, int rowIndex, String description) {
         this.tableId = tableId;
         this.fieldName = fieldName;

src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java

@@ -41,7 +41,7 @@ public class ProcessSingleFeedJob extends FeedVersionJob {
     public static boolean ENABLE_MTC_TRANSFORMATIONS = true;
 
     // Used in testing to skip validation and speed up response times.
-    public static boolean VALIDATE_MOBILITY_DATA = true;
+    public static boolean ENABLE_ADDITIONAL_VALIDATION = true;
 
     /**
      * Create a job for the given feed version.
@@ -131,8 +131,9 @@ public void jobLogic() {
 
         // Next, validate the feed.
         addNextJob(new ValidateFeedJob(feedVersion, owner, isNewVersion));
-        if (VALIDATE_MOBILITY_DATA) {
+        if (ENABLE_ADDITIONAL_VALIDATION) {
             addNextJob(new ValidateMobilityDataFeedJob(feedVersion, owner, isNewVersion));
+            addNextJob(new ValidateGtfsPlusFeedJob(feedVersion, owner, isNewVersion));
         }
 
         // We only need to snapshot the feed if there are transformations at the database level. In the case that there

src/main/java/com/conveyal/datatools/manager/jobs/ValidateGtfsPlusFeedJob.java

@@ -0,0 +1,70 @@
+package com.conveyal.datatools.manager.jobs;
+
+import com.conveyal.datatools.common.status.FeedVersionJob;
+import com.conveyal.datatools.manager.auth.Auth0UserProfile;
+import com.conveyal.datatools.manager.models.FeedVersion;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This job handles the GTFS+ validation of a given feed version. If the version is not new, it will simply
+ * replace the existing version with the version object that has updated validation info.
+ */
+public class ValidateGtfsPlusFeedJob extends FeedVersionJob {
+    public static final Logger LOG = LoggerFactory.getLogger(ValidateGtfsPlusFeedJob.class);
+
+    private final FeedVersion feedVersion;
+    private final boolean isNewVersion;
+
+    public ValidateGtfsPlusFeedJob(FeedVersion version, Auth0UserProfile owner, boolean isNewVersion) {
+        super(owner, "Validating GTFS+", JobType.VALIDATE_FEED);
+        feedVersion = version;
+        this.isNewVersion = isNewVersion;
+        status.update("Waiting to begin GTFS+ validation...", 0);
+    }
+
+    @Override
+    public void jobLogic () {
+        LOG.info("Running ValidateGtfsPlusFeedJob for {}", feedVersion.id);
+        feedVersion.validateGtfsPlus(status);
+    }
+
+    @Override
+    public void jobFinished () {
+        if (!status.error) {
+            if (parentJobId != null && JobType.PROCESS_FEED.equals(parentJobType)) {
+                // Validate stage is happening as part of an overall process feed job.
+                // At this point all GTFS data has been loaded and validated, so we record
+                // the FeedVersion into mongo.
+                // This happens here because otherwise we would have to wait for other jobs,
+                // such as BuildTransportNetwork, to finish. If those subsequent jobs fail,
+                // the version won't get loaded into MongoDB (even though it exists in postgres).
+                feedVersion.persistFeedVersionAfterValidation(isNewVersion);
+            }
+            status.completeSuccessfully("GTFS+ validation finished!");
+        } else {
+            // If the version was not stored successfully, call FeedVersion#delete to reset things to before the version
+            // was uploaded/fetched. Note: delete calls made to MongoDB on the version ID will not succeed, but that is
+            // expected.
+            feedVersion.delete();
+        }
+    }
+
+    /**
+     * Getter that allows a client to know the ID of the feed version that will be created as soon as the upload is
+     * initiated; however, we will not store the FeedVersion in the mongo application database until the upload and
+     * processing is completed. This prevents clients from manipulating GTFS data before it is entirely imported.
+     */
+    @JsonProperty
+    public String getFeedVersionId () {
+        return feedVersion.id;
+    }
+
+    @JsonProperty
+    public String getFeedSourceId () {
+        return feedVersion.parentFeedSource().id;
+    }
+
+
+}