feat: Add health check (#2247)

In order to effectively manage Kubernetes applications we want to have startup,
liveness, and readiness checks. This adds a package for exposing healthchecks that
include information about the runtime status of the application.

Author: blarghmatey

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install poetry
         uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
         with:
-          version: 1.8.5
+          version: 2.1.3
           virtualenvs-create: true
           virtualenvs-in-project: true
 

Dockerfile

@@ -31,7 +31,7 @@ ENV  \
   PYTHONDONTWRITEBYTECODE=1 \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
   POETRY_NO_INTERACTION=1 \
-  POETRY_VERSION=1.8.5 \
+  POETRY_VERSION=2.1.3 \
   POETRY_VIRTUALENVS_CREATE=true \
   POETRY_CACHE_DIR='/tmp/cache/poetry' \
   POETRY_HOME='/home/mitodl/.local' \

main/settings.py

@@ -130,7 +130,46 @@
     "data_fixtures",
     "vector_search",
     "mitol.scim.apps.ScimApp",
-)
+    "health_check",
+    "health_check.cache",
+    "health_check.contrib.migrations",
+    "health_check.contrib.celery_ping",
+    "health_check.contrib.redis",
+    "health_check.contrib.db_heartbeat",
+)
+
+HEALTH_CHECK = {
+    "SUBSETS": {
+        # The 'startup' subset includes checks that must pass before the application can
+        # start.
+        "startup": [
+            "MigrationsHealthCheck",  # Ensures database migrations are applied.
+            "CacheBackend",  # Verifies the cache backend is operational.
+            "RedisHealthCheck",  # Confirms Redis is reachable and functional.
+            "DatabaseHeartBeatCheck",  # Checks the database connection is alive.
+        ],
+        # The 'liveness' subset includes checks to determine if the application is
+        # running.
+        "liveness": ["DatabaseHeartBeatCheck"],  # Minimal check to ensure the app is
+        # alive.
+        # The 'readiness' subset includes checks to determine if the application is
+        # ready to serve requests.
+        "readiness": [
+            "CacheBackend",  # Ensures the cache is ready for use.
+            "RedisHealthCheck",  # Confirms Redis is ready for use.
+            "DatabaseHeartBeatCheck",  # Verifies the database is ready for queries.
+        ],
+        # The 'full' subset includes all available health checks for a comprehensive
+        # status report.
+        "full": [
+            "MigrationsHealthCheck",  # Ensures database migrations are applied.
+            "CacheBackend",  # Verifies the cache backend is operational.
+            "RedisHealthCheck",  # Confirms Redis is reachable and functional.
+            "DatabaseHeartBeatCheck",  # Checks the database connection is alive.
+            "CeleryPingHealthCheck",  # Verifies Celery workers are responsive.
+        ],
+    }
+}
 
 if not get_bool("RUN_DATA_MIGRATIONS", default=False):
     MIGRATION_MODULES = {"data_fixtures": None}
@@ -453,7 +492,6 @@
 }
 
 STATUS_TOKEN = get_string("STATUS_TOKEN", "")
-HEALTH_CHECK = ["CELERY", "REDIS", "POSTGRES", "OPEN_SEARCH"]
 
 GA_TRACKING_ID = get_string("GA_TRACKING_ID", "")
 GA_G_TRACKING_ID = get_string("GA_G_TRACKING_ID", "")

main/settings_celery.py

@@ -17,13 +17,11 @@
 
 DEV_ENV = get_bool("DEV_ENV", False)  # noqa: FBT003
 USE_CELERY = True
-CELERY_BROKER_URL = get_string("CELERY_BROKER_URL", get_string("REDISCLOUD_URL", None))
+REDIS_URL = get_string("REDIS_URL", get_string("REDISCLOUD_URL", None))
+CELERY_BROKER_URL = get_string("CELERY_BROKER_URL", REDIS_URL)
+CELERY_RESULT_BACKEND = get_string("CELERY_RESULT_BACKEND", REDIS_URL)
 CELERY_BEAT_SCHEDULER = RedBeatScheduler
-CELERY_REDBEAT_REDIS_URL = CELERY_BROKER_URL
 redbeat_redis_url = CELERY_BROKER_URL
-CELERY_RESULT_BACKEND = get_string(
-    "CELERY_RESULT_BACKEND", get_string("REDISCLOUD_URL", None)
-)
 CELERY_TASK_ALWAYS_EAGER = get_bool("CELERY_TASK_ALWAYS_EAGER", False)  # noqa: FBT003
 CELERY_TASK_EAGER_PROPAGATES = get_bool(
     "CELERY_TASK_EAGER_PROPAGATES",

main/urls.py

@@ -61,6 +61,7 @@
         re_path(r"^app", RedirectView.as_view(url=settings.APP_BASE_URL)),
         # Hijack
         re_path(r"^hijack/", include("hijack.urls", namespace="hijack")),
+        re_path(r"^health/", include("health_check.urls")),
     ]
     + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
     + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)

poetry.lock

@@ -8,108 +8,110 @@ license = "BSD-3"
 readme = "README.md"
 packages = []
 authors  = ["MIT ODL"]
+requires-poetry = ">2.1,<3"
 
 
 [tool.poetry.dependencies]
 python = "~3.12"
+Django = "4.2.21"
 attrs = "^25.0.0"
 base36 = "^0.1.1"
 beautifulsoup4 = "^4.8.2"
 boto3 = "^1.26.155"
 cairosvg = "2.7.1"
 celery = "^5.3.1"
+celery-redbeat = "^2.3.2"
 cffi = "^1.15.1"
 cryptography = "^44.0.0"
+dateparser = "^1.2.0"
+deepmerge = "^2.0"
 dj-database-url = "^2.0.0"
 dj-static = "^0.0.6"
-Django = "4.2.21"
 django-anymail = {extras = ["mailgun"], version = "^12.0"}
 django-bitfield = "^2.2.0"
 django-cache-memoize = "^0.2.0"
 django-cors-headers = "^4.0.0"
 django-filter = "^2.4.0"
 django-guardian = "^2.4.0"
+django-health-check = { git = "https://github.com/revsys/django-health-check", rev="9cfe2eaec5a15219513a36210b34875c03c64fe4" }  # pragma: allowlist secret
 django-hijack = "^3.4.1"
 django-imagekit = "^5.0.0"
 django-ipware = "^7.0.0"
 django-json-widget = "^2.0.0"
+django-oauth-toolkit = "^2.3.0"
 django-redis = "^5.2.0"
+django-scim2 = "^0.19.1"
 django-server-status = "^0.7.0"
 django-storages = "^1.13.2"
 djangorestframework = "^3.14.0"
 drf-jwt = "^1.19.2"
+drf-nested-routers = "^0.94.0"
 drf-spectacular = "^0.28.0"
 feedparser = "^6.0.10"
 google-api-python-client = "^2.89.0"
+html2text = "^2025.0.0"
 html5lib = "^1.1"
 ipython = "^9.0.0"
+isodate = "^0.7.2"
 jedi = "^0.19.0"
+langchain = "^0.3.11"
+langchain-experimental = "^0.3.4"
+langchain-openai = "^0.3.2"
+litellm = "1.66.1"
+llama-index = "^0.12.6"
+llama-index-agent-openai = "^0.4.1"
+llama-index-llms-openai = "^0.3.12"
 lxml = "^5.0.0"
+markdown = "^3.7"
 markdown2 = "^2.4.8"
+mitol-django-scim = "^2025.3.31"
+named-enum = "^1.4.0"
 nested-lookup = "^0.2.25"
+nh3 = "^0.2.14"
 ocw-data-parser = "^0.35.1"
+onnxruntime = "1.21.0"
+openai = "^1.55.3"
 opensearch-dsl = "^2.0.0"
 opensearch-py = "^2.0.0"
+opentelemetry-api = ">=1.31.0"
+opentelemetry-exporter-otlp = ">=1.31.0"
+opentelemetry-instrumentation-celery = ">=0.52b0"
+opentelemetry-instrumentation-django = ">=0.52b0"
+opentelemetry-instrumentation-psycopg = ">=0.52b0"
+opentelemetry-instrumentation-redis = ">=0.52b0"
+opentelemetry-instrumentation-requests = ">=0.52b0"
+opentelemetry-sdk = ">=1.31.0"
+pluggy = "^1.3.0"
+posthog = "^3.5.0"
+psycopg = "^3.2.4"
 psycopg2 = "^2.9.6"
+pycountry = "^24.6.1"
 pygithub = "^2.0.0"
+pyparsing = "^3.2.1"
+pytest-lazy-fixtures = "^1.1.1"
 python-dateutil = "^2.8.2"
 python-rapidjson = "^1.8"
 pyyaml = "^6.0.0"
+qdrant-client = {extras = ["fastembed"], version = "^1.12.0"}
 redis = "^5.0.0"
 requests = "^2.31.0"
-sentry-sdk = "2.25.1"
+retry2 = "^0.9.5"
+ruff = "0.11.5"
+selenium = "^4.30.0"
+sentry-sdk = "^2.25.1"
 social-auth-app-django = "^5.2.0"
+social-auth-core = {extras = ["openidconnect"], version = "^4.4.2"}
 static3 = "^0.7.0"
 tika = "^2.6.0"
+tiktoken = "^0.9.0"
 tldextract = "^5.0.0"
 toolz = "^1.0.0"
 ulid-py = "^0.2.0"
 urllib3 = "^2.0.0"
-uwsgi = "^2.0.21"
+uwsgi = "^2.0.29"
+uwsgitop = "^0.12"
 wrapt = "^1.14.1"
-social-auth-core = {extras = ["openidconnect"], version = "^4.4.2"}
-nh3 = "^0.2.14"
-retry2 = "^0.9.5"
-pluggy = "^1.3.0"
-named-enum = "^1.4.0"
-drf-nested-routers = "^0.94.0"
-django-scim2 = "^0.19.1"
-django-oauth-toolkit = "^2.3.0"
 youtube-transcript-api = "^1.0.0"
-posthog = "^3.5.0"
-ruff = "0.11.5"
-dateparser = "^1.2.0"
-uwsgitop = "^0.12"
-pytest-lazy-fixtures = "^1.1.1"
-pycountry = "^24.6.1"
-qdrant-client = {extras = ["fastembed"], version = "^1.12.0"}
-onnxruntime = "1.21.0"
-openai = "^1.55.3"
-litellm = "1.66.1"
-langchain = "^0.3.11"
-tiktoken = "^0.9.0"
-llama-index = "^0.12.6"
-llama-index-llms-openai = "^0.3.12"
-llama-index-agent-openai = "^0.4.1"
-langchain-experimental = "^0.3.4"
-langchain-openai = "^0.3.2"
-deepmerge = "^2.0"
-pyparsing = "^3.2.1"
-html2text = "^2025.0.0"
-markdown = "^3.7"
-isodate = "^0.7.2"
-selenium = "^4.30.0"
-mitol-django-scim = "^2025.3.31"
-opentelemetry-api = ">=1.31.0"
-opentelemetry-sdk = ">=1.31.0"
-opentelemetry-instrumentation-django = ">=0.52b0"
-opentelemetry-instrumentation-psycopg = ">=0.52b0"
-opentelemetry-instrumentation-redis = ">=0.52b0"
-opentelemetry-instrumentation-celery = ">=0.52b0"
-opentelemetry-instrumentation-requests = ">=0.52b0"
-opentelemetry-exporter-otlp = ">=1.31.0"
-psycopg = "^3.2.4"
-celery-redbeat = "^2.3.2"
 
 [tool.poetry.group.dev.dependencies]
 bpython = "^0.25"