Draft: Add BinaryApp application type and p47 example deployment

.devcontainer/devcontainer.json

@@ -29,7 +29,7 @@
     },
     "features": {
         // add in eternal history and other bash features
-        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1.0.0": {}
+        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {}
     },
     // Create the config folder for the bash-config feature
     "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
@@ -42,7 +42,11 @@
         "--device=/dev/fuse"
     ],
     "mounts": [
-        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached"
+        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached",
+        // these two mounts allow us to pick up cache and config folders that have
+        // been symlinked out of the home directory
+        "source=/dls/science/users/,target=/dls/science/users,type=bind,consistency=cached",
+        "source=/scratch/,target=/scratch,type=bind,consistency=cached"
     ],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",

.gitignore

@@ -72,3 +72,6 @@ lockfiles/
 
 # Demo output path for vscode tasks
 demo-output
+
+# vscode workspaces are local only
+*.code-workspace

src/deploy_tools/app_builder.py

@@ -1,6 +1,10 @@
 import uuid
+from hashlib import sha256
 from itertools import chain
 from pathlib import Path
+from urllib.request import urlretrieve
+
+from deploy_tools.models.binary_app import BinaryApp
 
 from .apptainer import create_sif_file
 from .layout import ModuleBuildLayout
@@ -27,6 +31,8 @@
                 self._create_apptainer_files(app, module)
             case ShellApp():
                 self._create_shell_file(app, module)
+            case BinaryApp():
+                self._create_binary_app(app, module)
 
     def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
         """Create apptainer entrypoints using a specified image and commands."""
@@ -44,6 +50,9 @@
             entrypoint_file = entrypoints_folder / entrypoint.name
 
             mounts = ",".join(chain(global_options.mounts, options.mounts)).strip()
+            host_binaries = " ".join(
+                chain(global_options.host_binaries, options.host_binaries)
+            ).strip()
 
             apptainer_args = f"{global_options.apptainer_args} {options.apptainer_args}"
             apptainer_args = apptainer_args.strip()
@@ -55,6 +64,7 @@
 
             params = {
                 "mounts": mounts,
+                "host_binaries": host_binaries,
                 "apptainer_args": apptainer_args,
                 "relative_sif_file": relative_sif_file,
                 "command": command,
@@ -96,3 +106,27 @@
             executable=True,
             create_parents=True,
         )
+
+    def _create_binary_app(self, app: BinaryApp, module: Module) -> None:
+        """
+        Download a URL, validate it against its sha256, make it executable
+        and add it to PATH
+        """
+        binary_folder = self._build_layout.get_binary_files_folder(
+            module.name, module.version
+        )
+        binary_path = binary_folder / app.name
+        binary_path.parent.mkdir(parents=True, exist_ok=True)
+        urlretrieve(app.url, binary_path)
+
+        h = sha256()
+        with open(binary_path, "rb") as fh:
+            while True:
+                data = fh.read(4096)
+                if len(data) == 0:
+                    break
+                h.update(data)
+        if h.hexdigest() != app.sha256:
+            raise RuntimeError(f"Downloaded Binary {app.url} digest failure")
+
+        binary_path.chmod(0o555)

src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml

@@ -0,0 +1,59 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: edge-containers-cli
+  version: "0.1"
+  description: ec command line tool for kubernetes IOCs
+
+  env_vars:
+    - name: EC_CLI_BACKEND
+      value: ARGOCD
+    - name: EC_LOG_URL
+      value: "https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}*"
+    # default configuration is for p47 for testing
+    - name: EC_TARGET
+      value: p47-beamline/p47
+    - name: EC_SERVICES_REPO
+      value: https://github.com/epics-containers/p47-services
+
+  applications:
+    - app_type: apptainer
+
+      container:
+        path: docker://ghcr.io/epics-containers/edge-containers-cli
+        version: "4.4.1"
+
+      entrypoints:
+        - name: ec
+          command: ec
+        # for debugging enter the container with bash shell
+        - name: ec-bash
+          command: bash
+
+      global_options:
+        mounts:
+          # places to get argocd and kubectl config from
+          - /dls/science/users/:/dls/science/users/
+          - /scratch:/scratch
+          - /dls_sw/apps:/dls_sw/apps
+        host_binaries:
+          - argocd
+          # for future expansion and to demonstrate it can be done
+          # load in the remaining kubernetes tools
+          - kubectl
+          - helm
+          - kubelogin
+          - klogout
+          - kustomize
+          - kubeseal
+          - kubectl-oidc_login
+
+    - app_type: binary
+      name: argocd
+      url: https://github.com/argoproj/argo-cd/releases/download/v2.14.10/argocd-linux-amd64
+      sha256: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa
+
+    - app_type: shell
+      name: ec-login
+      script:
+        - argocd login argocd.diamond.ac.uk --grpc-web --sso

src/deploy_tools/demo_configuration/p47/0.1.yaml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: p47
+  version: "0.1"
+  description: ec command line tool for kubernetes IOCs
+
+  env_vars:
+    - name: EC_TARGET
+      value: p47-beamline/p47
+    - name: EC_SERVICES_REPO
+      value: https://github.com/epics-containers/p47-services
+
+  dependencies:
+    - name: pollux
+      version: "local"
+    - name: edge-containers-cli
+      version: "0.1"
+
+  applications: []

src/deploy_tools/layout.py

@@ -8,6 +8,7 @@
     """
 
     ENTRYPOINTS_FOLDER = "entrypoints"
+    BINARIES_FOLDER = "binaries"
 
     MODULE_SNAPSHOT_FILENAME = "module.yaml"
     MODULEFILE_FILENAME = "modulefile"
@@ -21,6 +22,9 @@
     def get_entrypoints_folder(self, name: str, version: str) -> Path:
         return self.get_module_folder(name, version) / self.ENTRYPOINTS_FOLDER
 
+    def get_binaries_folder(self, name: str, version: str) -> Path:
+        return self.get_module_folder(name, version) / self.BINARIES_FOLDER
+
     def get_modulefile(self, name: str, version: str) -> Path:
         return self.get_module_folder(name, version) / self.MODULEFILE_FILENAME
 
@@ -37,10 +41,14 @@
     """
 
     SIF_FILES_FOLDER = "sif_files"
+    BINARY_FILES_FOLDER = "binaries"
 
     def get_sif_files_folder(self, name: str, version: str) -> Path:
         return self.get_module_folder(name, version) / self.SIF_FILES_FOLDER
 
+    def get_binary_files_folder(self, name: str, version: str) -> Path:
+        return self.get_module_folder(name, version) / self.BINARY_FILES_FOLDER
+
     @property
     def build_root(self):
         return self._root
@@ -73,6 +81,9 @@
     def get_entrypoints_folder(self, name: str, version: str) -> Path:
         return self._modules_layout.get_entrypoints_folder(name, version)
 
+    def get_binaries_folder(self, name: str, version: str) -> Path:
+        return self._modules_layout.get_binaries_folder(name, version)
+
     def get_modulefiles_root(self, from_deprecated: bool = False) -> Path:
         return (
             self.deprecated_modulefiles_root

src/deploy_tools/models/apptainer_app.py

@@ -8,6 +8,7 @@ class EntrypointOptions(ParentModel):
     apptainer_args: str = ""
     command_args: str = ""
     mounts: Sequence[str] = []
+    host_binaries: Sequence[str] = []
 
 
 class Entrypoint(ParentModel):

src/deploy_tools/models/binary_app.py

@@ -0,0 +1,17 @@
+from typing import Literal
+
+from .parent import ParentModel
+
+
+class BinaryApp(ParentModel):
+    """
+    Represents a standalone Binary application.
+
+    This will fetch a standalone binary, validate its sha256 and add its
+    location to that path.
+    """
+
+    app_type: Literal["binary"]
+    name: str
+    url: str
+    sha256: str

src/deploy_tools/models/module.py

@@ -4,10 +4,13 @@
 from pydantic import Field
 
 from .apptainer_app import ApptainerApp
+from .binary_app import BinaryApp
 from .parent import ParentModel
 from .shell_app import ShellApp
 
-Application = Annotated[ApptainerApp | ShellApp, Field(..., discriminator="app_type")]
+Application = Annotated[
+    ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type")
+]
 
 DEVELOPMENT_VERSION = "dev"
 

src/deploy_tools/models/schemas/deployment.json

@@ -1,6 +1,6 @@
 {
   "$defs": {
-    "Apptainer": {
+    "ApptainerApp": {
       "additionalProperties": false,
       "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.",
       "properties": {
@@ -24,7 +24,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -33,7 +34,38 @@
         "container",
         "entrypoints"
       ],
-      "title": "Apptainer",
+      "title": "ApptainerApp",
+      "type": "object"
+    },
+    "BinaryApp": {
+      "additionalProperties": false,
+      "description": "Represents a standalone Binary application.\n\nThis will fetch a standalone binary, validate its sha256 and add its\nlocation to that path.",
+      "properties": {
+        "app_type": {
+          "const": "binary",
+          "title": "App Type",
+          "type": "string"
+        },
+        "name": {
+          "title": "Name",
+          "type": "string"
+        },
+        "url": {
+          "title": "Url",
+          "type": "string"
+        },
+        "sha256": {
+          "title": "Sha256",
+          "type": "string"
+        }
+      },
+      "required": [
+        "app_type",
+        "name",
+        "url",
+        "sha256"
+      ],
+      "title": "BinaryApp",
       "type": "object"
     },
     "ContainerImage": {
@@ -98,7 +130,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -128,6 +161,14 @@
           },
           "title": "Mounts",
           "type": "array"
+        },
+        "host_binaries": {
+          "default": [],
+          "items": {
+            "type": "string"
+          },
+          "title": "Host Binaries",
+          "type": "array"
         }
       },
       "title": "EntrypointOptions",
@@ -197,17 +238,21 @@
           "items": {
             "discriminator": {
               "mapping": {
-                "apptainer": "#/$defs/Apptainer",
-                "shell": "#/$defs/Shell"
+                "apptainer": "#/$defs/ApptainerApp",
+                "binary": "#/$defs/BinaryApp",
+                "shell": "#/$defs/ShellApp"
               },
               "propertyName": "app_type"
             },
             "oneOf": [
               {
-                "$ref": "#/$defs/Apptainer"
+                "$ref": "#/$defs/ApptainerApp"
+              },
+              {
+                "$ref": "#/$defs/ShellApp"
               },
               {
-                "$ref": "#/$defs/Shell"
+                "$ref": "#/$defs/BinaryApp"
               }
             ]
           },
@@ -281,7 +326,7 @@
       },
       "type": "object"
     },
-    "Shell": {
+    "ShellApp": {
       "additionalProperties": false,
       "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.",
       "properties": {
@@ -307,7 +352,7 @@
         "name",
         "script"
       ],
-      "title": "Shell",
+      "title": "ShellApp",
       "type": "object"
     }
   },