liveness check: use periodic check-ins. this is missing a migration

Author: codl

model.py

@@ -367,3 +367,7 @@ class MastodonInstance(db.Model):
 
     def bump(self, value=1):
         self.popularity = (self.popularity or 10) + value
+
+class WorkerCheckin(db.Model, TimestampMixin):
+    __tablename__ = 'worker_checkins'
+    id = db.Column(db.Integer, primary_key=True)

routes/api.py

@@ -1,10 +1,11 @@
 from app import app, db, imgproxy
 from libforget.auth import require_auth_api, get_viewer
 from flask import jsonify, redirect, make_response, request, Response
-from model import Account
+from model import Account, WorkerCheckin
 import libforget.settings
 import libforget.json
 import random
+from datetime import datetime, timedelta
 
 @app.route('/api/health_check') # deprecated 2021-03-12
 @app.route('/api/status_check')
@@ -19,14 +20,23 @@ def api_status_check():
     except Exception:
         return ('Redis bad', 500)
 
-    if db.session.execute(db.text("""
-        SELECT 1 FROM accounts
-        WHERE last_delete > now() - '60 minutes'::INTERVAL
-        OR last_fetch > now() - '60 minutes'::INTERVAL
-        OR last_refresh > now() - '60 minutes'::INTERVAL
-        LIMIT 1;
-    """)).fetchone() is None:
-        return ('Celery stalled', 500)
+    CHECKIN_EVENTS = 5
+    CHECKIN_PERIOD = timedelta(minutes=10)
+    # sorry about the obtuse variable names, this trips if the frequency is
+    # lower than events/period
+    checkin_count = db.session.query(WorkerCheckin)\
+            .filter(WorkerCheckin.created_at > db.func.now() - CHECKIN_PERIOD)\
+            .count()
+    if checkin_count < events:
+        return ('Celery slow, {} check-ins in {}'.format(
+            checkin_count, CHECKIN_PERIOD
+        ), 500)
+
+    CHECKIN_LATENESS_THRESHOLD = timedelta(minutes=5)
+    checkin = db.session.query(WorkerCheckin.created_at)\
+            .order_by(db.desc(WorkerCheckin.created_at)).first()
+    if checkin + CHECKIN_LATENESS_THRESHOLD < datetime.utcnow():
+        return ('Celery late, last check-in was {}'.format(checkin), 500)
 
     return 'OK'
 

tasks.py

@@ -2,7 +2,7 @@
 from app import app as flaskapp
 from app import db
 from model import Session, Account, TwitterArchive, Post, OAuthToken,\
-                  MastodonInstance
+                  MastodonInstance, WorkerCheckin
 import libforget.twitter
 import libforget.mastodon
 from datetime import timedelta, datetime, timezone
@@ -386,6 +386,11 @@ def periodic_cleanup():
         you have restored access and you can now re-enable Forget if you wish.
         """.format(service=account.service.capitalize())
 
+    # delete worker check-ins after 48 hours
+    (WorkerCheckin.query.filter(
+        WorkerCheckin.updated_at < (db.func.now() - timedelta(hours=48)))
+     .delete(synchronize_session=False))
+
     db.session.commit()
 
 
@@ -474,13 +479,19 @@ def update_mastodon_instances_popularity():
     })
     db.session.commit()
 
+@app.task
+def report_in():
+    db.session.add(WorkerCheckin())
+    db.session.commit()
+
 
 app.add_periodic_task(40, queue_fetch_for_most_stale_accounts)
 app.add_periodic_task(9, queue_deletes)
 app.add_periodic_task(6, refresh_account_with_oldest_post)
 app.add_periodic_task(50, refresh_account_with_longest_time_since_refresh)
 app.add_periodic_task(300, periodic_cleanup)
 app.add_periodic_task(300, update_mastodon_instances_popularity)
+app.add_periodic_task(60, report_in)
 
 if __name__ == '__main__':
     app.worker_main()