Human in the loop

compute_worker/compute_worker.py

@@ -314,6 +314,9 @@ def run_wrapper(run_args):
         run.prepare()
         run.start()
         if run.is_scoring:
+            if run.human_in_the_loop:
+                run.wait_for_human_validation()
+                run._update_status(SubmissionStatus.FINISHED)
             run.push_scores()
         run.push_output()
     except DockerImagePullException as e:
@@ -470,6 +473,7 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        self.human_in_the_loop = run_args.get("human_in_the_loop", False)
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -1445,11 +1449,55 @@ def start(self):
                 )
                 # Raise so upstream marks failed immediately
                 raise SubmissionException("Child task failed or non-zero return code")
-            self._update_status(SubmissionStatus.FINISHED)
+
+            if not self.human_in_the_loop:
+                self._update_status(SubmissionStatus.FINISHED)
 
         else:
             self._update_status(SubmissionStatus.SCORING)
 
+    def wait_for_human_validation(self):
+        container_output_dir = self.output_dir
+        host_output_dir = self._get_host_path(self.output_dir)
+
+        scores_path = os.path.join(host_output_dir, "scores.json")
+        if not os.path.exists(os.path.join(container_output_dir, "scores.json")):
+            scores_path = os.path.join(host_output_dir, "scores.txt")
+
+        approved_container = os.path.join(container_output_dir, "hitl_approved")
+        rejected_container = os.path.join(container_output_dir, "hitl_rejected")
+        approved_host = os.path.join(host_output_dir, "hitl_approved")
+        rejected_host = os.path.join(host_output_dir, "hitl_rejected")
+
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect the scores file:")
+        logger.info(f"  cat {scores_path}")
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+
+        poll_interval = 3
+        max_wait = 60 * 60 * 24
+
+        elapsed = 0
+        while elapsed < max_wait:
+            if os.path.exists(approved_container):  # ← poll le chemin conteneur
+                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                return
+            if os.path.exists(rejected_container):  # ← poll le chemin conteneur
+                raise SubmissionException(
+                    f"HITL: scores rejected by the compute node operator "
+                    f"(submission {self.submission_id})"
+                )
+            time.sleep(poll_interval)
+            elapsed += poll_interval
+
+        raise SubmissionException(
+            f"HITL: 24h timeout reached without validation "
+            f"(submission {self.submission_id})"
+        )
+
     def push_scores(self):
         """This is only ran at the end of the scoring step"""
         # POST to some endpoint:

src/apps/api/serializers/competitions.py

@@ -272,7 +272,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def validate_phases(self, phases):
@@ -417,7 +418,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def get_leaderboards(self, instance):

src/apps/competitions/models.py

@@ -89,6 +89,7 @@ class Competition(models.Model):
 
     # If true, forum is enabled (default=True)
     forum_enabled = models.BooleanField(default=True)
+    enable_human_in_the_loop = models.BooleanField(default=False)
 
     def __str__(self):
         return f"competition-{self.title}-{self.pk}-{self.competition_type}"

src/apps/competitions/tasks.py

@@ -59,6 +59,7 @@
     "contact_email",
     "fact_sheet",
     "forum_enabled",
+    "enable_human_in_the_loop"
 ]
 
 TASK_FIELDS = [
@@ -134,6 +135,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
+        "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop,
     }
 
     if (
@@ -212,6 +214,9 @@ def _send_to_compute_worker(submission, is_scoring):
     time_padding = 60 * 20  # 20 minutes
     time_limit = submission.phase.execution_time_limit + time_padding
 
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        time_limit = 60 * 60 * 25
+
     if (
         submission.phase.competition.queue
     ):  # if the competition is running on a custom queue, not the default queue

src/static/riot/competitions/editor/_competition_details.tag

@@ -211,6 +211,22 @@
             </sup>
         </div>
 
+        <!--  Human in the Loop  -->
+        <div class="field">
+            <label>Human in the Loop</label>
+            <div class="ui checkbox">
+                <label>Enable Human in the Loop validation</label>
+                <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
+            </div>
+            <sup>
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform"
+                        data-inverted=""
+                        data-position="bottom center">
+                    <i class="help icon circle"></i>
+                </span>
+            </sup>
+        </div>
+
         <!--  Public submissions  -->
         <div class="field">
             <label>Public Submissions</label>
@@ -314,6 +330,7 @@
             self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
+            self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()
@@ -452,6 +469,7 @@
             self.refs.auto_run_submissions.checked = competition.auto_run_submissions
             self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public
             self.refs.forum_enabled.checked = competition.forum_enabled
+            self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop
             self.refs.make_programs_available.checked = competition.make_programs_available
             self.refs.make_input_data_available.checked = competition.make_input_data_available
             $(self.refs.docker_image).val(competition.docker_image)