healthcheck: Don't report error for tasks that are probably running

cmsmanage/views.py

@@ -42,35 +42,43 @@ def _check_clusters() -> CheckResult:
         return CheckResultOk(f"{len(happy_clusters)}/{len(clusters)} clusters up")
 
 
-def _check_tasks() -> Iterable[CheckResult]:
-    now = timezone.now()
-    for schedule in Schedule.objects.all():
+def _check_task(now: datetime.datetime, schedule: Schedule) -> CheckResult:
     if not schedule.task:
-            yield CheckResultFailure(f"Scheduled task {schedule} has never run!")
+        return CheckResultFailure(f"Scheduled task {schedule} has never run!")
 
     else:
         try:
             task = Task.objects.get(id=schedule.task)
         except Task.DoesNotExist:
-                yield CheckResultFailure(
-                    f"Scheduled task {schedule}'s last task doesn't exist!"
+            if now - schedule.next_run > datetime.timedelta(minutes=5):
+                return CheckResultFailure(
+                    f"Scheduled task {schedule}'s last task doesn't exist, and is probably not still running!"
+                )
+            else:
+                return CheckResultOk(
+                    f"Schedule {schedule} has no task, but probably running now"
                 )
-                continue
 
         if not task.success:
-                yield CheckResultFailure(
+            return CheckResultFailure(
                 f"Scheduled task {schedule} failed at {task.started}"
             )
         elif now - schedule.next_run > datetime.timedelta(hours=2):
-                yield CheckResultFailure(
+            return CheckResultFailure(
                 f"Scheduled task {schedule} stale, last run at {task.started}"
             )
         else:
-                yield CheckResultOk(
+            return CheckResultOk(
                 f"Scheduled task {schedule} ok, last run at {task.started}"
             )
 
 
+def _check_tasks() -> Iterable[CheckResult]:
+    now = timezone.now()
+    for schedule in Schedule.objects.all():
+        yield _check_task(now, schedule)
+
+
 def healthcheck(request: HttpRequest):
     checks: list[CheckResult] = [_check_clusters(), *_check_tasks()]