Feature/task-and-output-test-coverage

src/wdlci/cli/__main__.py

@@ -5,6 +5,9 @@
 from wdlci.cli.submit import submit_handler
 from wdlci.cli.monitor import monitor_handler
 from wdlci.cli.cleanup import cleanup_handler
+from wdlci.cli.detect_task_and_output_coverage import (
+    detect_task_and_output_coverage_handler,
+)
 from wdlci.utils.ordered_group import OrderedGroup
 
 remove_option = click.option(
@@ -100,3 +103,10 @@ def cleanup(**kwargs):
     """Clean Workbench namespace of transient artifacts"""
 
     cleanup_handler(kwargs)
+
+
+@main.command
+def detect_task_and_output_coverage(**kwargs):
+    """Outputs how much coverage each task and output has, and which tasks/outputs have no associated tests"""
+
+    detect_task_and_output_coverage_handler(kwargs)

src/wdlci/cli/detect_changes.py

@@ -15,18 +15,13 @@ def detect_changes_handler(kwargs):
 
         changeset = Changeset()
 
-        tasks_without_tests = []
-
         for workflow, workflow_config in config.file.workflows.items():
             doc = WDL.load(workflow)
 
             for task in doc.tasks:
                 task_name = task.name
                 task_digest = task.digest
 
-                if not workflow_config.tasks[task_name].tests:
-                    tasks_without_tests.append(task_name)
-
                 if task_name in workflow_config.tasks:
                     previous_digest = workflow_config.tasks[task_name].digest
                     if (
@@ -40,7 +35,6 @@ def detect_changes_handler(kwargs):
                     print(f"New task detected [{workflow} - {task_name}]")
                     workflow_change = changeset.add_workflow_change(workflow)
                     workflow_change.add_task_change(task_name)
-        print(f"The following tasks have no tests: {', '.join(tasks_without_tests)}")
 
         if len(changeset.workflow_changes) == 0:
             print("No new or modified tasks detected")

src/wdlci/cli/detect_task_and_output_coverage.py

@@ -0,0 +1,84 @@
+import sys
+import WDL
+from wdlci.config import Config
+from wdlci.config.config_file import WorkflowTaskConfig, WorkflowTaskTestConfig
+from wdlci.exception.wdl_test_cli_exit_exception import WdlTestCliExitException
+
+
+def detect_task_and_output_coverage_handler(kwargs):
+    try:
+        Config.load(kwargs)
+        config = Config.instance()
+
+        tasks_without_tests = []
+        outputs_without_tests = []
+
+        task_coverage = {}
+        output_coverage = {}
+
+        for workflow, workflow_config in config.file.workflows.items():
+            doc = WDL.load(workflow)
+
+            for task in doc.tasks:
+                task_name = task.name
+                # task_test_configs is a list of WorkflowTaskTestConfig objects,
+                # which contain inputs and output_tests as attributes
+                task_test_configs = workflow_config.tasks[task_name].tests
+
+                if not task_test_configs:
+                    tasks_without_tests.append(task_name)
+
+                task_coverage_count = 0
+
+                for task_test_config in task_test_configs:
+                    output_tests_dict = task_test_config.output_tests
+                    output_coverage[task_name] = {}
+
+                    for output_key, output_value in output_tests_dict.items():
+                        if (
+                            isinstance(output_value, dict)
+                            and "test_tasks" in output_value
+                        ):
+                            output_tests_list = output_value["test_tasks"]
+                            # Assign the length of test tasks to the key associated with a
+                            # given output in a nested structure, where each task has a
+                            # dictionary of outputs and their task counts
+                            output_coverage[task_name][output_key] = len(
+                                output_tests_list
+                            )
+                            task_coverage_count += len(output_tests_list)
+
+                            if not output_tests_list:
+                                outputs_without_tests.append(output_key)
+
+                    task_coverage[task_name] = task_coverage_count
+
+        print("Task Coverage:")
+        for task_name, tests_count in task_coverage.items():
+            print(f"\t{task_name}: {tests_count}")
+
+        print("\nOutput coverage:")
+        for task_name, output_tests_dict in output_coverage.items():
+            print(f"\tTask name: {task_name}")
+            for output_name, tests_count in output_tests_dict.items():
+                print(
+                    f"\t\tOutput name: {output_name}\n\t\t\tnumber of tests: {tests_count}"
+                )
+
+        if tasks_without_tests and outputs_without_tests:
+            print(
+                f"[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n"
+                + f"Additionally, the following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n"
+            )
+        elif tasks_without_tests:
+            print(
+                f"[WARN]: The following tasks have no tests:\n{', '.join(tasks_without_tests)}\n\n"
+            )
+        elif outputs_without_tests:
+            print(
+                f"[WARN]: The following outputs have no tests:\n{', '.join(outputs_without_tests)}\n\n"
+            )
+
+    except WdlTestCliExitException as e:
+        print(f"exiting with code {e.exit_code}, message: {e.message}")
+        sys.exit(e.exit_code)

src/wdlci/wdl_tests/check_zip.wdl

@@ -0,0 +1,47 @@
+version 1.0
+
+# Check integrity of ZIP file
+# Input type: ZIP file
+
+task check_zip {
+	input {
+		File current_run_output
+		File validated_output
+	}
+
+	Int disk_size = ceil(size(current_run_output, "GB") + size(validated_output, "GB") + 50)
+
+	command <<<
+		set -euo pipefail
+
+		err() {
+			message=$1
+
+			echo -e "[ERROR] $message" >&2
+		}
+
+		if ! zip -T ~{validated_output}; then
+			err "Validated file: [~{basename(validated_output)}] did not pass zip check"
+			exit 1
+		else
+			if ! zip -T ~{current_run_output}; then
+				err "Current run file: [~{basename(current_run_output)}] did not pass zip check"
+				exit 1
+			else
+				echo "Current run file: [~{basename(current_run_output)}] passed zip check"
+			fi
+		fi
+	>>>
+
+	output {
+	}
+
+	runtime {
+		docker: "dnastack/dnastack-wdl-ci-tools:0.0.1"
+		cpu: 1
+		memory: "3.75 GB"
+		disk: disk_size + " GB"
+		disks: "local-disk " + disk_size + " HDD"
+		preemptible: 1
+	}
+}