Combine existing PRS and add fully working P47 beamline example

.devcontainer/devcontainer.json

@@ -29,7 +29,7 @@
     },
     "features": {
         // add in eternal history and other bash features
-        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1.0.0": {}
+        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {}
     },
     // Create the config folder for the bash-config feature
     "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
@@ -42,7 +42,11 @@
         "--device=/dev/fuse"
     ],
     "mounts": [
-        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached"
+        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached",
+        // these two mounts allow us to pick up cache and config folders that have
+        // been symlinked out of the home directory
+        "source=/dls/science/users/,target=/dls/science/users,type=bind,consistency=cached",
+        "source=/scratch/,target=/scratch,type=bind,consistency=cached"
     ],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",

.github/workflows/_test.yml

@@ -51,7 +51,13 @@ jobs:
           pip-install: ".[dev]"
 
       - name: Run tests
-        run: tox -e tests
+        run: |
+          sudo apt update
+          sudo apt install -y software-properties-common
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt update
+          sudo apt install -y apptainer
+          tox -e tests
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

.gitignore

@@ -72,3 +72,6 @@ lockfiles/
 
 # Demo output path for vscode tasks
 demo-output
+
+# vscode workspaces are local only
+*.code-workspace

src/deploy_tools/app_builder.py

@@ -1,6 +1,10 @@
+import hashlib
 import uuid
 from itertools import chain
 from pathlib import Path
+from urllib.request import urlretrieve
+
+from deploy_tools.models.binary_app import BinaryApp, HashType
 
 from .apptainer import create_sif_file
 from .layout import ModuleBuildLayout
@@ -27,6 +31,8 @@ def create_application_files(self, app: Application, module: Module):
                 self._create_apptainer_files(app, module)
             case ShellApp():
                 self._create_shell_file(app, module)
+            case BinaryApp():
+                self._create_binary_file(app, module)
 
     def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
         """Create apptainer entrypoints using a specified image and commands."""
@@ -44,6 +50,9 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
             entrypoint_file = entrypoints_folder / entrypoint.name
 
             mounts = ",".join(chain(global_options.mounts, options.mounts)).strip()
+            host_binaries = " ".join(
+                chain(global_options.host_binaries, options.host_binaries)
+            ).strip()
 
             apptainer_args = f"{global_options.apptainer_args} {options.apptainer_args}"
             apptainer_args = apptainer_args.strip()
@@ -55,6 +64,7 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
 
             params = {
                 "mounts": mounts,
+                "host_binaries": host_binaries,
                 "apptainer_args": apptainer_args,
                 "relative_sif_file": relative_sif_file,
                 "command": command,
@@ -96,3 +106,39 @@ def _create_shell_file(self, app: ShellApp, module: Module) -> None:
             executable=True,
             create_parents=True,
         )
+
+    ALL_READ_EXECUTE_PERMISSIONS = 0o555
+
+    def _create_binary_file(self, app: BinaryApp, module: Module) -> None:
+        """
+        Download a URL, validate it against its sha256, make it executable
+        and add it to PATH
+        """
+        binary_folder = self._build_layout.get_entrypoints_folder(
+            module.name, module.version
+        )
+        binary_path = binary_folder / app.name
+        binary_path.parent.mkdir(parents=True, exist_ok=True)
+        urlretrieve(app.url, binary_path)
+
+        match app.hash_type:
+            case HashType.SHA256:
+                h = hashlib.sha256()
+            case HashType.SHA512:
+                h = hashlib.sha512()
+            case HashType.MD5:
+                h = hashlib.md5()
+            case HashType.NONE:
+                h = None
+
+        if h is not None:
+            with open(binary_path, "rb") as fh:
+                while True:
+                    data = fh.read(4096)
+                    if len(data) == 0:
+                        break
+                    h.update(data)
+            if h.hexdigest() != app.hash:
+                raise AppBuilderError(f"Downloaded Binary {app.url} hash check failed")
+
+        binary_path.chmod(self.ALL_READ_EXECUTE_PERMISSIONS)

src/deploy_tools/demo_configuration/argocd/0.1.yaml

@@ -0,0 +1,13 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: argocd
+  version: "0.1"
+  description: Adds argocd command line tool
+
+  applications:
+    - app_type: binary
+      name: argocd
+      url: https://github.com/argoproj/argo-cd/releases/download/v2.14.10/argocd-linux-amd64
+      hash_type: sha256
+      hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa

src/deploy_tools/demo_configuration/ec-i13-1/0.1.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: ec-i13-1
+  version: "0.1"
+  description: Command line setup for i13-1 beamline
+
+  dependencies:
+    - name: edge-containers-cli
+      version: "0.1"
+    - name: k8s-i13-1
+      version: local
+
+  applications:
+    - app_type: shell
+      name: ec
+      script:
+        - ec-apptainer --target i13-1-beamline/i13-1 --repo https://github.com/epics-containers/i13-1-services

src/deploy_tools/demo_configuration/ec-i13-1/dev.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: ec-i13-1
+  version: dev
+  description: Command line setup for i13-1 beamline
+
+  dependencies:
+    - name: edge-containers-cli
+      version: "0.1"
+    - name: k8s-i13-1
+      version: local
+
+  applications:
+    - app_type: shell
+      name: ec
+      script:
+        - ec-apptainer --target i13-1-beamline/i13-1 --repo https://gitlab.diamond.ac.uk/controls/containers/beamline/i13-1-services.git ${@}

src/deploy_tools/demo_configuration/ec-p47/0.1.yaml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: ec-p47
+  version: "0.1"
+  description: Command line setup for p47 beamline
+
+  env_vars:
+    - name: EC_TARGET
+      value: p47-beamline/p47
+    - name: EC_SERVICES_REPO
+      value: https://github.com/epics-containers/p47-services
+
+  dependencies:
+    - name: edge-containers-cli
+      version: "0.1"
+    - name: pollux
+      version: local
+
+  applications: []

src/deploy_tools/demo_configuration/ec-p47/dev.yaml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: ec-p47
+  version: dev
+  description: Command line setup for p47 beamline
+
+  dependencies:
+    - name: edge-containers-cli
+      version: "0.1"
+    - name: pollux
+      version: local
+
+  applications:
+    - app_type: shell
+      name: ec
+      script:
+        - ec-apptainer --target p47-beamline/p47 --repo https://github.com/epics-containers/p47-services ${@}

src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml

@@ -0,0 +1,51 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: edge-containers-cli
+  version: "0.1"
+  description: ec command line tool for kubernetes IOCs
+
+  dependencies:
+    - name: argocd
+      version: "0.1"
+
+  env_vars:
+    - name: EC_CLI_BACKEND
+      value: ARGOCD
+    - name: EC_LOG_URL
+      value: "https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}*"
+
+  applications:
+    - app_type: apptainer
+
+      container:
+        path: docker://ghcr.io/epics-containers/edge-containers-cli
+        version: "4.4.1"
+
+      entrypoints:
+        - name: ec-apptainer
+          command: ec
+        # for debugging enter the container with bash shell
+        - name: ec-bash
+          command: bash
+
+      global_options:
+        mounts:
+          # places to get argocd and kubectl config from
+          - /dls/science/users/:/dls/science/users/
+          - /scratch:/scratch
+          - /dls_sw/apps:/dls_sw/apps
+        host_binaries:
+          - argocd
+          - kubectl
+          - helm
+          - kubelogin
+          - klogout
+          - kustomize
+          - kubeseal
+          - kubectl-oidc_login
+
+    - app_type: shell
+      name: ec-login
+      script:
+        - argocd login argocd.diamond.ac.uk --grpc-web --sso

src/deploy_tools/models/apptainer_app.py

@@ -1,13 +1,31 @@
 from collections.abc import Sequence
 from typing import Literal
 
+from pydantic import Field
+
 from .parent import ParentModel
 
 
 class EntrypointOptions(ParentModel):
-    apptainer_args: str = ""
-    command_args: str = ""
-    mounts: Sequence[str] = []
+    apptainer_args: str = Field(
+        description="Apptainer arguments to pass when launching the container",
+        default="",
+    )
+    command_args: str = Field(
+        description="Arguments to pass to command entrypoint",
+        default="",
+    )
+    mounts: Sequence[str] = Field(
+        description="A list of mount points to add to the container in the form of "
+        "'host_path:container_path'",
+        default=[],
+    )
+    host_binaries: Sequence[str] = Field(
+        description="A list of host binaries to mount into the container. These are "
+        "discovered on the host using the current PATH and are mounted into the "
+        "container at /usr/bin/<binary_name>.",
+        default=[],
+    )
 
 
 class Entrypoint(ParentModel):

src/deploy_tools/models/binary_app.py

@@ -0,0 +1,44 @@
+from enum import StrEnum
+from typing import Literal
+
+from pydantic import Field
+
+from .parent import ParentModel
+
+
+class HashType(StrEnum):
+    """
+    Type of hash to use for the binary.
+    """
+
+    SHA256 = "sha256"
+    SHA512 = "sha512"
+    MD5 = "md5"
+    NONE = "none"
+
+
+class BinaryApp(ParentModel):
+    """
+    Represents a standalone Binary application.
+
+    This will fetch a standalone binary, validate its hash and add its
+    location to that path.
+    """
+
+    app_type: Literal["binary"]
+    name: str = Field(
+        ...,
+        description="Binary filename to use locally",
+    )
+    url: str = Field(
+        ...,
+        description="URL to download the binary from.",
+    )
+    hash_type: HashType = Field(
+        ...,
+        description="Type of hash used to check the binary.",
+    )
+    hash: str = Field(
+        "",
+        description="Hash to verify binary integrity",
+    )

src/deploy_tools/models/module.py

@@ -3,11 +3,15 @@
 
 from pydantic import Field
 
+from deploy_tools.models.binary_app import BinaryApp
+
 from .apptainer_app import ApptainerApp
 from .parent import ParentModel
 from .shell_app import ShellApp
 
-Application = Annotated[ApptainerApp | ShellApp, Field(..., discriminator="app_type")]
+Application = Annotated[
+    ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type")
+]
 
 DEVELOPMENT_VERSION = "dev"
 

src/deploy_tools/models/parent.py

@@ -2,6 +2,15 @@
 
 
 class ParentModel(BaseModel):
-    """Will forbid any extra parameters being provided in any subclass."""
+    """
+    Provides Model Config for all Pydantic models in this project:
 
-    model_config = ConfigDict(extra="forbid")
+      forbid:          forbid any extra parameters being provided in any subclass.
+      use_enum_values: use the enum value only when serializing the model,
+                       this means the yaml serializer can work with enums
+    """
+
+    model_config = ConfigDict(
+        extra="forbid",
+        use_enum_values=True,
+    )