Add tasks and management command for scim sync

Author: rhysyngsun

docker-compose.yml

@@ -9,6 +9,11 @@ services:
     ports:
     - "${POSTGRES_PORT:-55432}:5432"
 
+  redis:
+    image: redis
+    ports:
+    - "${REDIS_PORT:-36379}:6379"
+
   shell:
     build:
       target: uv

pyproject.toml

@@ -6,7 +6,7 @@ authors = [
     { name = "MIT Open Learning Engineering", email = "odl-devops@mit.edu" }
 ]
 dependencies = [
-  "mitol-django-common",
+  "mitol-django-common[celery]",
   "mitol-django-mail",
   'mitol-django-authentication[touchstone]; python_version < "3.13"',
   'mitol-django-authentication; python_version >= "3.13"',
@@ -22,7 +22,7 @@ dependencies = [
   "mitol-django-payment_gateway",
   "mitol-django-uvtestapp",
   "mitol-django-transcoding",
-  "mitol-django-scim",
+  "mitol-django-scim[celery]",
   "mitol-django-apigateway",
 ]
 readme = "README.md"
@@ -64,13 +64,15 @@ dev-dependencies = [
     "semver",
     "toml",
     "typing-extensions",
+    "pytest-responses>=0.5.1",
+    "more-itertools>=10.6.0",
 ]
 
 [tool.hatch.metadata]
 allow-direct-references = true
 
 [tool.uv.workspace]
-members = ["src/*"]
+members = ["src/*", "testapp"]
 
 [tool.uv.pip]
 no-binary = ["lxml", "xmlsec"]
@@ -214,7 +216,7 @@ lint.ignore = [
     "RET507",
     "RET508",
     "RUF012",
-    "UP007"
+    "UP007",
 ]
 lint.typing-modules = ["colour.hints"]
 

src/common/mitol/common/models.py

@@ -11,6 +11,7 @@
 from django.db import transaction
 from django.db.models import (
     PROTECT,
+    CharField,
     DateTimeField,
     ForeignKey,
     JSONField,
@@ -243,3 +244,21 @@ def _clone(self):
             self._prefetch_generic_related_lookups
         )
         return c
+
+
+class UserGlobalIdMixin:
+    """
+    Mixin that adds a standard global_id definition.
+
+    The `global_id` field points to the SSO ID for the user (so, usually the Keycloak
+    ID, which is a UUID). We store it as a string in case the SSO source changes.
+    We allow a blank value so we can have out-of-band users - we may want a
+     Django user that's not connected to an SSO user, for instance.
+    """
+
+    global_id = CharField(
+        max_length=36,
+        blank=True,
+        default="",
+        help_text="The SSO ID (usually a Keycloak UUID) for the user.",
+    )

src/common/mitol/common/settings/celery.py

@@ -0,0 +1,10 @@
+"""Celery settings"""
+
+from mitol.common.envs import get_string
+
+MITOL_CELERY_APP_INSTANCE_PATH = get_string(
+    name="MITOL_CELERY_APP_INSTANCE_PATH",
+    default="main.celery.app",
+    description="Path to the celery app instance",
+    required=True,
+)

src/common/mitol/common/utils/celery.py

@@ -0,0 +1,18 @@
+from celery import Celery
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.utils.module_loading import import_string
+
+
+def get_celery_app() -> Celery:
+    """Get the celery app for the project we're running in"""
+    app = import_string(settings.MITOL_CELERY_APP_INSTANCE_PATH)
+
+    if not isinstance(app, Celery):
+        msg = (
+            "Setting MITOL_CELERY_APP_INSTANCE_PATH does not reference "
+            "an instance of `celery.Celery`"
+        )
+        raise ImproperlyConfigured(msg)
+
+    return app

src/common/pyproject.toml

@@ -3,20 +3,25 @@ name = "mitol-django-common"
 description = "MIT Open Learning django app extensions for common utilities"
 version = "2025.4.2"
 dependencies = [
-"django-redis~=5.0",
-"django-stubs>=1.13.1",
-"django-webpack-loader>=0.7.0",
-"django>=3.0",
-"factory-boy~=3.2",
-"pytest>=7,<9",
-"pytz>=2020.4",
-"requests>=2.20.0",
-"typing-extensions",
+  "django-redis~=5.0",
+  "django-stubs>=1.13.1",
+  "django-webpack-loader>=0.7.0",
+  "django>=3.0",
+  "factory-boy~=3.2",
+  "pytest>=7,<9",
+  "pytz>=2020.4",
+  "requests>=2.20.0",
+  "typing-extensions",
 ]
 readme = "README.md"
 license = "BSD-3-Clause"
 requires-python = ">=3.10"
 
+[project.optional-dependencies]
+celery = [
+  "celery>=5.5.0",
+]
+
 [tool.bumpver]
 current_version = "2025.4.2"
 version_pattern = "YYYY.MM.DD[.INC0]"

src/scim/README.md

@@ -2,14 +2,17 @@
 
 ## Prerequisites
 
-- You need the following a local [Keycloak](https://www.keycloak.org/) instance running. Note which major version you are running (should be at least 26.x).
+- You need the following a local [Keycloak](https://www.keycloak.org/) instance
+  running. Note which major version you are running (should be at least 26.x).
   - You should have custom user profile fields setup on your `olapps` realm:
     - `fullName`: required, otherwise defaults
     - `emailOptIn`: defaults
 
 ## Install the scim-for-keycloak plugin
 
-Sign up for an account on https://scim-for-keycloak.de and follow the instructions here: https://scim-for-keycloak.de/documentation/installation/install
+Sign up for an account on https://scim-for-keycloak.de and follow the
+instructions here:
+https://scim-for-keycloak.de/documentation/installation/install
 
 ## Configure SCIM
 
@@ -20,7 +23,8 @@ In the SCIM admin console, do the following:
 - In django-admin, go to OAuth Toolkit and create a new access token
 - Go to Remote SCIM Provider
 - Click the `+` button
-- Specify a base URL for your learn API backend: `http://<IP_OR_HOSTNAME>:8063/scim/v2/`
+- Specify a base URL for your learn API backend:
+  `http://<IP_OR_HOSTNAME>:8063/scim/v2/`
 - At the bottom of the page, click "Use default configuration"
 - Add a new authentication method:
   - Type: Long Life Bearer Token
@@ -30,6 +34,8 @@ In the SCIM admin console, do the following:
   - Add an attribute named `emailOptIn` with the following settings:
     - Type: integer
     - Custom Attribute Name: `emailOptIn`
+  - Add an `emailVerified` attribute (using the option specific to this in the
+    dropdown)
 - On the Realm Assignments tab, assign to the `olapps` realm
 - Go to the Synchronization tab and perform one:
   - Identifier attribute: email

src/scim/mitol/scim/adapters.py

@@ -131,6 +131,7 @@ def from_dict(self, d):
         self.obj.last_name = d.get("name", {}).get("familyName", "")
         self.obj.scim_username = d.get("userName")
         self.obj.scim_external_id = d.get("externalId")
+        self.obj.global_id = self.obj.scim_external_id
 
     def _save_user(self):
         self.obj.save()