From a828f712f4e95bdf17dd5b277b2c2e16f6dcf94d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 13:18:49 +0200 Subject: [PATCH 01/48] =?UTF-8?q?=E2=9C=A8=20[Models]=20Update=20project?= =?UTF-8?q?=20and=20node=20models=20with=20improved=20field=20annotations?= =?UTF-8?q?=20and=20examples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/projects_nodes.py | 3 --- .../models_library/rpc/webserver/projects.py | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 66683369f35..a2fd95fa740 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -78,7 +78,6 @@ class NodeState(BaseModel): description="true if the node's outputs need to be re-computed", ), ] = True - dependencies: Annotated[ set[NodeID], Field( @@ -86,7 +85,6 @@ class NodeState(BaseModel): description="contains the node inputs dependencies if they need to be computed first", ), ] = DEFAULT_FACTORY - current_status: Annotated[ RunningState, Field( @@ -94,7 +92,6 @@ class NodeState(BaseModel): alias="currentStatus", ), ] = RunningState.NOT_STARTED - progress: Annotated[ float | None, Field( diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 010c1c41d84..0a12e5fc69b 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -54,6 +54,28 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } ) + # Specific to jobs + job_parent_resource_name: str + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + nodes_examples = Node.model_json_schema()["examples"] + schema.update( + { + "examples": [ + { + "uuid": "12345678-1234-5678-1234-123456789012", + "name": "My project", + "description": "My project description", + "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, + "creation_date": "2023-01-01T00:00:00Z", + "last_change_date": "2023-01-01T00:00:00Z", + "job_parent_resource_name": "solvers/foo/release/1.2.3", + }, + ] + } + ) + model_config = ConfigDict( extra="forbid", populate_by_name=True, From 5a967e16c944537ba0de8f23457c52696e098955 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:57:26 +0200 Subject: [PATCH 02/48] fixes tests --- .../models_library/rpc/webserver/projects.py | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 0a12e5fc69b..10d760ac4d0 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -35,28 +35,6 @@ class ProjectJobRpcGet(BaseModel): # Specific to jobs job_parent_resource_name: str - @staticmethod - def _update_json_schema_extra(schema: JsonDict) -> None: - nodes_examples = Node.model_json_schema()["examples"] - schema.update( - { - "examples": [ - { - "uuid": "12345678-1234-5678-1234-123456789012", - "name": "My project", - "description": "My project description", - "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, - "creation_at": "2023-01-01T00:00:00Z", - "modified_at": "2023-01-01T00:00:00Z", - "job_parent_resource_name": "solvers/foo/release/1.2.3", - }, - ] - } - ) - - # Specific to jobs - job_parent_resource_name: str - @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: nodes_examples = Node.model_json_schema()["examples"] From 901b9dcbfa238a3d2bd0b0977a290fb8922ab556 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:09:42 +0200 Subject: [PATCH 03/48] fixes tests --- .../src/models_library/rpc/webserver/projects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 10d760ac4d0..010c1c41d84 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -46,8 +46,8 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "name": "My project", "description": "My project description", "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, - "creation_date": "2023-01-01T00:00:00Z", - "last_change_date": "2023-01-01T00:00:00Z", + "creation_at": "2023-01-01T00:00:00Z", + "modified_at": "2023-01-01T00:00:00Z", "job_parent_resource_name": "solvers/foo/release/1.2.3", }, ] From 0b536a902185cc58df5f4f1cdd2580a6903f3683 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:50:43 +0200 Subject: [PATCH 04/48] mypy --- packages/models-library/src/models_library/projects_nodes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index a2fd95fa740..66683369f35 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -78,6 +78,7 @@ class NodeState(BaseModel): description="true if the node's outputs need to be re-computed", ), ] = True + dependencies: Annotated[ set[NodeID], Field( @@ -85,6 +86,7 @@ class NodeState(BaseModel): description="contains the node inputs dependencies if they need to be computed first", ), ] = DEFAULT_FACTORY + current_status: Annotated[ RunningState, Field( @@ -92,6 +94,7 @@ class NodeState(BaseModel): alias="currentStatus", ), ] = RunningState.NOT_STARTED + progress: Annotated[ float | None, Field( From 090e64e7de227813642314291e710519ed6971f2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:46:11 +0200 Subject: [PATCH 05/48] =?UTF-8?q?=E2=9C=A8=20Add=20endpoint=20to=20list=20?= =?UTF-8?q?all=20jobs=20for=20released=20solvers=20(paginated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/routes/solvers_jobs_getters.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index a0e751fa2dc..d98a02b3581 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -56,6 +56,7 @@ from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor from ..dependencies.services import get_api_client from ..dependencies.webserver_http import AuthSession, get_webserver_session +from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import ( FMSG_CHANGELOG_NEW_IN_VERSION, FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT, @@ -116,9 +117,27 @@ **DEFAULT_BACKEND_SERVICE_STATUS_CODES, } + router = APIRouter() +@router.get( + "/-/releases/-/jobs", + response_model=Page[Job], + description="List of all jobs created for any released solver (paginated)", + include_in_schema=API_SERVER_DEV_FEATURES_ENABLED, +) +async def list_all_solvers_jobs( + user_id: Annotated[PositiveInt, Depends(get_current_user_id)], + page_params: Annotated[PaginationParams, Depends()], + solver_service: Annotated[SolverService, Depends(SolverService)], + webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], + url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], + product_name: Annotated[str, Depends(get_product_name)], +): + raise NotImplementedError + + @router.get( "/{solver_key:path}/releases/{version}/jobs", response_model=list[Job], From f392f5fdbbe576408762785842f1daa5d675c0c2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 11:59:12 +0200 Subject: [PATCH 06/48] =?UTF-8?q?=E2=9C=A8=20Add=20method=20to=20list=20al?= =?UTF-8?q?l=20solver=20jobs=20with=20pagination=20in=20SolverService?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 44 ++++++++++++++++++- .../services_rpc/wb_api_server.py | 19 ++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 5aa76086396..0cba13f32a1 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -4,7 +4,12 @@ from fastapi import Depends from models_library.basic_types import VersionStr from models_library.products import ProductName -from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.rest_pagination import ( + MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, + PageOffsetInt, +) +from models_library.rpc.webserver.projects import PageRpcProjectRpcGet +from models_library.rpc_pagination import PageLimitInt from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease from models_library.users import UserID @@ -12,15 +17,22 @@ from .models.schemas.solvers import Solver, SolverKeyId from .services_rpc.catalog import CatalogService +from .services_rpc.wb_api_server import WbApiRpcClient DEFAULT_PAGINATION_LIMIT = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1 class SolverService: _catalog_service: CatalogService + _webserver_client: WbApiRpcClient - def __init__(self, catalog_service: Annotated[CatalogService, Depends()]): + def __init__( + self, + catalog_service: Annotated[CatalogService, Depends()], + wb_api_client: Annotated[WbApiRpcClient, Depends()], + ): self._catalog_service = catalog_service + self._webserver_client = wb_api_client async def get_solver( self, @@ -70,3 +82,31 @@ async def get_latest_release( ) return Solver.create_from_service(service) + + async def list_all_solvers_jobs( + self, + *, + user_id: UserID, + product_name: ProductName, + offset: PageOffsetInt = 0, + limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, + ) -> PageRpcProjectRpcGet: + """Lists solver jobs for a user with pagination + + Args: + user_id: The ID of the user + product_name: The product name + offset: Pagination offset + limit: Pagination limit + job_parent_resource_name_filter: Optional filter for job parent resource name + + Returns: + Paginated response with projects marked as jobs + """ + return await self._webserver_client.list_projects_marked_as_jobs( + product_name=product_name, + user_id=user_id, + offset=offset, + limit=limit, + job_parent_resource_name_filter="solvers", # TODO: use a constant from models_library + ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index ca771d913b1..670cc03952c 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -219,6 +219,25 @@ async def mark_project_as_job( job_parent_resource_name=job_parent_resource_name, ) + @_exception_mapper(rpc_exception_map={}) + async def list_projects_marked_as_jobs( + self, + *, + product_name: ProductName, + user_id: UserID, + offset: int = 0, + limit: int = 50, + job_parent_resource_name_filter: str | None = None, + ): + return await projects_rpc.list_projects_marked_as_jobs( + rpc_client=self._client, + product_name=product_name, + user_id=user_id, + offset=offset, + limit=limit, + job_parent_resource_name_filter=job_parent_resource_name_filter, + ) + def setup(app: FastAPI, rabbitmq_rmp_client: RabbitMQRPCClient): wb_api_rpc_client = WbApiRpcClient(_client=rabbitmq_rmp_client) From 5551ea3706135fea2b2a5b57cf7d84abd9ef3800 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 12:19:13 +0200 Subject: [PATCH 07/48] rename --- .../src/simcore_service_api_server/_service_solvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 0cba13f32a1..31edcf48d5f 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -83,7 +83,7 @@ async def get_latest_release( return Solver.create_from_service(service) - async def list_all_solvers_jobs( + async def list_jobs( self, *, user_id: UserID, From 82ddf4bff94ecc0c695e9520efb878bd37fb5cb0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 12:47:34 +0200 Subject: [PATCH 08/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20imports=20in=20sol?= =?UTF-8?q?ver=20job=20modules=20and=20add=20new=20model=20converters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simcore_service_api_server/_service_job.py | 2 +- .../api/routes/solvers_jobs.py | 6 +++--- .../api/routes/solvers_jobs_getters.py | 2 +- .../api/routes/studies_jobs.py | 2 +- .../solver_job_models_converters.py | 15 ++++++++------- .../services_http/jobs.py | 2 +- ...> test_models_solver_job_models_converters.py} | 2 +- 7 files changed, 16 insertions(+), 15 deletions(-) rename services/api-server/src/simcore_service_api_server/{services_http => models}/solver_job_models_converters.py (95%) rename services/api-server/tests/unit/{test_services_solver_job_models_converters.py => test_models_solver_job_models_converters.py} (99%) diff --git a/services/api-server/src/simcore_service_api_server/_service_job.py b/services/api-server/src/simcore_service_api_server/_service_job.py index 1b7cd79a317..61d43d8a034 100644 --- a/services/api-server/src/simcore_service_api_server/_service_job.py +++ b/services/api-server/src/simcore_service_api_server/_service_job.py @@ -14,7 +14,7 @@ from .models.schemas.jobs import Job, JobInputs from .models.schemas.programs import Program from .models.schemas.solvers import Solver -from .services_http.solver_job_models_converters import ( +from .models.solver_job_models_converters import ( create_job_from_project, create_new_project_for_job, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 37c97d45e86..f331e4da872 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -27,11 +27,11 @@ JobStatus, ) from ...models.schemas.solvers import Solver, SolverKeyId -from ...services_http.director_v2 import DirectorV2Api -from ...services_http.jobs import replace_custom_metadata, start_project, stop_project -from ...services_http.solver_job_models_converters import ( +from ...models.solver_job_models_converters import ( create_jobstatus_from_task, ) +from ...services_http.director_v2 import DirectorV2Api +from ...services_http.jobs import replace_custom_metadata, start_project, stop_project from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index d98a02b3581..a10f7f25108 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -41,13 +41,13 @@ WalletGetWithAvailableCreditsLegacy, ) from ...models.schemas.solvers import SolverKeyId +from ...models.solver_job_models_converters import create_job_from_project from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import ( get_custom_metadata, raise_if_job_not_associated_with_solver, ) from ...services_http.log_streaming import LogDistributor, LogStreamer -from ...services_http.solver_job_models_converters import create_job_from_project from ...services_http.solver_job_outputs import ResultsTypes, get_solver_output_results from ...services_http.storage import StorageApi, to_file_api_model from ..dependencies.application import get_reverse_url_mapper diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 0043b5daa70..8f6edba0cae 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -29,6 +29,7 @@ JobStatus, ) from ...models.schemas.studies import JobLogsMap, Study, StudyID +from ...models.solver_job_models_converters import create_jobstatus_from_task from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import ( get_custom_metadata, @@ -36,7 +37,6 @@ start_project, stop_project, ) -from ...services_http.solver_job_models_converters import create_jobstatus_from_task from ...services_http.storage import StorageApi from ...services_http.study_job_models_converters import ( create_job_from_study, diff --git a/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py similarity index 95% rename from services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py rename to services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py index 81ea1683a78..9c4e67d1f5e 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py @@ -15,11 +15,13 @@ from models_library.basic_types import KeyIDStr from models_library.projects import Project from models_library.projects_nodes import InputID +from models_library.rpc.webserver.projects import ProjectRpcGet from pydantic import HttpUrl, TypeAdapter -from ..models.domain.projects import InputTypes, Node, SimCoreFileLink -from ..models.schemas.files import File -from ..models.schemas.jobs import ( +from ..services_http.director_v2 import ComputationTaskGet +from .domain.projects import InputTypes, Node, SimCoreFileLink +from .schemas.files import File +from .schemas.jobs import ( ArgumentTypes, Job, JobInputs, @@ -29,9 +31,8 @@ get_runner_url, get_url, ) -from ..models.schemas.programs import Program -from ..models.schemas.solvers import Solver -from .director_v2 import ComputationTaskGet +from .schemas.programs import Program +from .schemas.solvers import Solver # UTILS ------ _BASE_UUID = uuid.UUID("231e13db-6bc6-4f64-ba56-2ee2c73b9f09") @@ -182,7 +183,7 @@ def create_new_project_for_job( def create_job_from_project( *, solver_or_program: Solver | Program, - project: ProjectGet | Project, + project: ProjectRpcGet | ProjectGet | Project, url_for: Callable[..., HttpUrl], ) -> Job: """ diff --git a/services/api-server/src/simcore_service_api_server/services_http/jobs.py b/services/api-server/src/simcore_service_api_server/services_http/jobs.py index ed2ef50d588..5b7b0fd441b 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/jobs.py +++ b/services/api-server/src/simcore_service_api_server/services_http/jobs.py @@ -17,8 +17,8 @@ JobPricingSpecification, JobStatus, ) +from ..models.solver_job_models_converters import create_jobstatus_from_task from .director_v2 import DirectorV2Api -from .solver_job_models_converters import create_jobstatus_from_task from .webserver import AuthSession _logger = logging.getLogger(__name__) diff --git a/services/api-server/tests/unit/test_services_solver_job_models_converters.py b/services/api-server/tests/unit/test_models_solver_job_models_converters.py similarity index 99% rename from services/api-server/tests/unit/test_services_solver_job_models_converters.py rename to services/api-server/tests/unit/test_models_solver_job_models_converters.py index 1e926f09c86..3aba0cf9ecf 100644 --- a/services/api-server/tests/unit/test_services_solver_job_models_converters.py +++ b/services/api-server/tests/unit/test_models_solver_job_models_converters.py @@ -10,7 +10,7 @@ from simcore_service_api_server.models.schemas.files import File from simcore_service_api_server.models.schemas.jobs import ArgumentTypes, Job, JobInputs from simcore_service_api_server.models.schemas.solvers import Solver -from simcore_service_api_server.services_http.solver_job_models_converters import ( +from simcore_service_api_server.models.solver_job_models_converters import ( create_job_from_project, create_job_inputs_from_node_inputs, create_jobstatus_from_task, From 2b0358d074356b94d755926c88d3107b49656766 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 12:48:12 +0200 Subject: [PATCH 09/48] draft --- .../_service_solvers.py | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 31edcf48d5f..04612a597dc 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -6,14 +6,15 @@ from models_library.products import ProductName from models_library.rest_pagination import ( MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, + PageMetaInfoLimitOffset, PageOffsetInt, ) -from models_library.rpc.webserver.projects import PageRpcProjectRpcGet from models_library.rpc_pagination import PageLimitInt from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease from models_library.users import UserID from packaging.version import Version +from simcore_service_api_server.models.schemas.jobs import Job from .models.schemas.solvers import Solver, SolverKeyId from .services_rpc.catalog import CatalogService @@ -29,10 +30,10 @@ class SolverService: def __init__( self, catalog_service: Annotated[CatalogService, Depends()], - wb_api_client: Annotated[WbApiRpcClient, Depends()], + webserver_client: Annotated[WbApiRpcClient, Depends()], ): self._catalog_service = catalog_service - self._webserver_client = wb_api_client + self._webserver_client = webserver_client async def get_solver( self, @@ -61,6 +62,7 @@ async def get_latest_release( solver_key: SolverKeyId, product_name: str, ) -> Solver: + # TODO: Mads, this is not necessary. The first item is the latest! service_releases: list[ServiceRelease] = [] for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): releases, page_meta = await self._catalog_service.list_release_history( @@ -90,23 +92,32 @@ async def list_jobs( product_name: ProductName, offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, - ) -> PageRpcProjectRpcGet: - """Lists solver jobs for a user with pagination - - Args: - user_id: The ID of the user - product_name: The product name - offset: Pagination offset - limit: Pagination limit - job_parent_resource_name_filter: Optional filter for job parent resource name - - Returns: - Paginated response with projects marked as jobs - """ - return await self._webserver_client.list_projects_marked_as_jobs( + ) -> tuple[list[Job], PageMetaInfoLimitOffset]: + """Lists all solver jobs for a user with pagination""" + + # NOTE: perhaps we should get comp_tasks instead of projects! or a combinatino of both? + # I need inputs_checksum and job_parent_source_name! + + projects_page = await self._webserver_client.list_projects_marked_as_jobs( product_name=product_name, user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter="solvers", # TODO: use a constant from models_library + job_parent_resource_name_filter="solvers", # TODO: project shouldr eturn parent resource name and workbench ) + + solver = None # TODO + url_for = None # TODO + + jobs: list[Job] = [ + Job( + id=prj.uuid, + name=prj.name, + inputs_checksum=prj.inputs_checksum, + created_at=prj.creation_date, # type: ignore[arg-type] + runner_name=prj.job_parent_resource_name, + # TODO: url_ parts missing + ) + for prj in projects_page.data + ] + return jobs, projects_page.meta From 7b6b759bc2d522f6651223ddf51c1099b71d724d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 17:13:21 +0200 Subject: [PATCH 10/48] =?UTF-8?q?=E2=9C=A8=20Update=20get=5Fsolver=20funct?= =?UTF-8?q?ion=20signature=20to=20improve=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simcore_service_api_server/api/routes/solvers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index da286e491cd..03cc24757f5 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -156,7 +156,7 @@ async def get_solver( solver_service: Annotated[SolverService, Depends(SolverService)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], -) -> Solver: +): """Gets latest release of a solver""" # IMPORTANT: by adding /latest, we avoid changing the order of this entry in the router list # otherwise, {solver_key:path} will override and consume any of the paths that follow. From 84ff3dbcce00b582c7879c4ca57d53de983bdc0e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 17:46:11 +0200 Subject: [PATCH 11/48] drafting tests --- .../_service_solvers.py | 5 ++- .../tests/unit/test_service_solvers.py | 41 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 services/api-server/tests/unit/test_service_solvers.py diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 04612a597dc..a78a95ed711 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -14,8 +14,9 @@ from models_library.services_history import ServiceRelease from models_library.users import UserID from packaging.version import Version -from simcore_service_api_server.models.schemas.jobs import Job +from .api.dependencies.webserver_rpc import get_wb_api_rpc_client +from .models.schemas.jobs import Job from .models.schemas.solvers import Solver, SolverKeyId from .services_rpc.catalog import CatalogService from .services_rpc.wb_api_server import WbApiRpcClient @@ -30,7 +31,7 @@ class SolverService: def __init__( self, catalog_service: Annotated[CatalogService, Depends()], - webserver_client: Annotated[WbApiRpcClient, Depends()], + webserver_client: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], ): self._catalog_service = catalog_service self._webserver_client = webserver_client diff --git a/services/api-server/tests/unit/test_service_solvers.py b/services/api-server/tests/unit/test_service_solvers.py new file mode 100644 index 00000000000..0d79dd4ff8d --- /dev/null +++ b/services/api-server/tests/unit/test_service_solvers.py @@ -0,0 +1,41 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import pytest +from models_library.products import ProductName +from models_library.users import UserID +from pytest_mock import MockerFixture, MockType +from simcore_service_api_server._service_solvers import SolverService +from simcore_service_api_server.models.schemas.solvers import Solver +from simcore_service_api_server.services_rpc.catalog import CatalogService +from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient + + +@pytest.fixture +def solver_service( + mocker: MockerFixture, + mocked_rpc_catalog_service_api: dict[str, MockType], +) -> SolverService: + return SolverService( + catalog_service=CatalogService(client=mocker.MagicMock()), + webserver_client=WbApiRpcClient(_client=mocker.MagicMock()), + ) + + +async def test_get_solver( + solver_service: SolverService, + mocked_rpc_catalog_service_api: dict[str, MockType], + product_name: ProductName, + user_id: UserID, +): + solver = await solver_service.get_solver( + user_id=user_id, + name="simcore/services/comp/solver-1", + version="1.0.0", + product_name=product_name, + ) + + assert isinstance(solver, Solver) + mocked_rpc_catalog_service_api["get_service"].assert_called_once() From ca46e7f79881bd708a51f48f9d1f5e90afdda183 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 17:51:19 +0200 Subject: [PATCH 12/48] =?UTF-8?q?=E2=9C=A8=20Rename=20mocked=5Frpc=5Fcatal?= =?UTF-8?q?og=5Fservice=5Fapi=20to=20mocked=5Fcatalog=5Frpc=5Fapi=20for=20?= =?UTF-8?q?consistency=20across=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/api-server/tests/unit/test_service_solvers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-server/tests/unit/test_service_solvers.py b/services/api-server/tests/unit/test_service_solvers.py index 0d79dd4ff8d..e213f041578 100644 --- a/services/api-server/tests/unit/test_service_solvers.py +++ b/services/api-server/tests/unit/test_service_solvers.py @@ -16,7 +16,7 @@ @pytest.fixture def solver_service( mocker: MockerFixture, - mocked_rpc_catalog_service_api: dict[str, MockType], + mocked_catalog_rpc_api: dict[str, MockType], ) -> SolverService: return SolverService( catalog_service=CatalogService(client=mocker.MagicMock()), @@ -26,7 +26,7 @@ def solver_service( async def test_get_solver( solver_service: SolverService, - mocked_rpc_catalog_service_api: dict[str, MockType], + mocked_catalog_rpc_api: dict[str, MockType], product_name: ProductName, user_id: UserID, ): @@ -38,4 +38,4 @@ async def test_get_solver( ) assert isinstance(solver, Solver) - mocked_rpc_catalog_service_api["get_service"].assert_called_once() + mocked_catalog_rpc_api["get_service"].assert_called_once() From 2782d396a80803960beeae87dc38d213d8bdd2d4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 18:03:12 +0200 Subject: [PATCH 13/48] Extends fake --- .../src/models_library/rpc/webserver/projects.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 010c1c41d84..9f7b54cdd86 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -54,6 +54,22 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } ) + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.udpate( + { + "examples": [ + { + "uuid": "12345678-1234-5678-1234-123456789012", + "name": "My project", + "description": "My project description", + "creation_date": "2023-01-01T00:00:00Z", + "last_change_date": "2023-01-01T00:00:00Z", + }, + ] + } + ) + model_config = ConfigDict( extra="forbid", populate_by_name=True, From 44f558b869b4418cd3cac2f2f681f3de08a648b9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 18:06:56 +0200 Subject: [PATCH 14/48] drafting tests --- .../tests/unit/test_service_solvers.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/services/api-server/tests/unit/test_service_solvers.py b/services/api-server/tests/unit/test_service_solvers.py index e213f041578..2c2f5727649 100644 --- a/services/api-server/tests/unit/test_service_solvers.py +++ b/services/api-server/tests/unit/test_service_solvers.py @@ -17,6 +17,7 @@ def solver_service( mocker: MockerFixture, mocked_catalog_rpc_api: dict[str, MockType], + mocked_webserver_rpc_api: dict[str, MockType], ) -> SolverService: return SolverService( catalog_service=CatalogService(client=mocker.MagicMock()), @@ -39,3 +40,43 @@ async def test_get_solver( assert isinstance(solver, Solver) mocked_catalog_rpc_api["get_service"].assert_called_once() + + +async def test_list_jobs( + solver_service: SolverService, + mocked_webserver_rpc_api: dict[str, MockType], + product_name: ProductName, + user_id: UserID, +): + # Test default parameters + jobs, page_meta = await solver_service.list_jobs( + user_id=user_id, + product_name=product_name, + ) + assert isinstance(jobs, list) + mocked_webserver_rpc_api["list_projects_marked_as_jobs"].assert_called_once_with( + product_name=product_name, + user_id=user_id, + offset=0, + limit=999, + job_parent_resource_name_filter="solvers", + ) + assert page_meta.total >= 0 + assert page_meta.limit == 999 + assert page_meta.offset == 0 + + # Test with explicit pagination + mocked_webserver_rpc_api["list_projects_marked_as_jobs"].reset_mock() + jobs, page_meta = await solver_service.list_jobs( + user_id=user_id, + product_name=product_name, + offset=10, + limit=20, + ) + mocked_webserver_rpc_api["list_projects_marked_as_jobs"].assert_called_once_with( + product_name=product_name, + user_id=user_id, + offset=10, + limit=20, + job_parent_resource_name_filter="solvers", + ) From 2bd54614b466a5c844a76b71065b5928350ec7f4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 23 Apr 2025 19:35:14 +0200 Subject: [PATCH 15/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20project-related=20?= =?UTF-8?q?models=20and=20RPCs=20for=20consistency=20and=20clarity=20-=20R?= =?UTF-8?q?ename=20ProjectRpcGet=20to=20ProjectJobRpcGet=20-=20Update=20jo?= =?UTF-8?q?b-related=20models=20to=20include=20workbench=20and=20job=5Fpar?= =?UTF-8?q?ent=5Fresource=5Fname=20-=20Adjust=20job=20creation=20logic=20t?= =?UTF-8?q?o=20use=20updated=20models=20-=20Modify=20tests=20to=20reflect?= =?UTF-8?q?=20model=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models_library/rpc/webserver/projects.py | 6 +- .../_service_solvers.py | 42 ++++++++----- .../models/schemas/jobs.py | 61 +++++++++++++------ .../models/solver_job_models_converters.py | 11 +--- 4 files changed, 77 insertions(+), 43 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 9f7b54cdd86..ab0a16fe0d7 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -54,9 +54,11 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } ) + job_parent_resource_name: str + @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: - schema.udpate( + schema.update( { "examples": [ { @@ -65,6 +67,8 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "description": "My project description", "creation_date": "2023-01-01T00:00:00Z", "last_change_date": "2023-01-01T00:00:00Z", + "job_parent_resource_name": "solvers/foo/release/1.2.3", + "workbench": {}, }, ] } diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index a78a95ed711..952029dfb3d 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -4,6 +4,7 @@ from fastapi import Depends from models_library.basic_types import VersionStr from models_library.products import ProductName +from models_library.projects_nodes import Node from models_library.rest_pagination import ( MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, PageMetaInfoLimitOffset, @@ -16,8 +17,9 @@ from packaging.version import Version from .api.dependencies.webserver_rpc import get_wb_api_rpc_client -from .models.schemas.jobs import Job +from .models.schemas.jobs import JobInputs, JobModel from .models.schemas.solvers import Solver, SolverKeyId +from .models.solver_job_models_converters import create_job_inputs_from_node_inputs from .services_rpc.catalog import CatalogService from .services_rpc.wb_api_server import WbApiRpcClient @@ -93,7 +95,7 @@ async def list_jobs( product_name: ProductName, offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, - ) -> tuple[list[Job], PageMetaInfoLimitOffset]: + ) -> tuple[list[JobModel], PageMetaInfoLimitOffset]: """Lists all solver jobs for a user with pagination""" # NOTE: perhaps we should get comp_tasks instead of projects! or a combinatino of both? @@ -107,18 +109,28 @@ async def list_jobs( job_parent_resource_name_filter="solvers", # TODO: project shouldr eturn parent resource name and workbench ) - solver = None # TODO - url_for = None # TODO - - jobs: list[Job] = [ - Job( - id=prj.uuid, - name=prj.name, - inputs_checksum=prj.inputs_checksum, - created_at=prj.creation_date, # type: ignore[arg-type] - runner_name=prj.job_parent_resource_name, - # TODO: url_ parts missing + jobs: list[JobModel] = [] + + for prj in projects_page.data: + + assert len(prj.workbench) == 1, "Expected only one solver node in workbench" + + solver_node: Node = next(iter(prj.workbench.values())) + job_inputs: JobInputs = create_job_inputs_from_node_inputs( + inputs=solver_node.inputs or {} ) - for prj in projects_page.data - ] + assert prj.job_parent_resource_name # nosec + + jobs.append( + JobModel( + id=prj.uuid, + name=JobModel.compose_resource_name( + prj.job_parent_resource_name, prj.uuid + ), + inputs_checksum=job_inputs.compute_checksum(), + created_at=prj.creation_date, + runner_name=prj.job_parent_resource_name, + ) + ) + return jobs, projects_page.meta diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index 19899f57ef6..9e3e11c2a4d 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -27,6 +27,7 @@ ValidationInfo, field_validator, ) +from pydantic.types import JsonDict from servicelib.logging_utils import LogLevelInt, LogMessageStr from starlette.datastructures import Headers @@ -237,7 +238,7 @@ class JobMetadata(BaseModel): # "jobs/c2789bd2-7385-4e00-91d3-2f100df41185" -class Job(BaseModel): +class JobModel(BaseModel): id: JobID name: RelativeResourceName @@ -249,6 +250,41 @@ class Job(BaseModel): ..., description="Runner that executes job" ) + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.udpate( + { + "examples": [ + { + "id": "f622946d-fd29-35b9-a193-abdd1095167c", + "name": "solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c", + "runner_name": "solvers/isolve/releases/1.3.4", + "inputs_checksum": "12345", + "created_at": "2021-01-22T23:59:52.322176", + }, + ] + } + ) + + model_config = ConfigDict( + json_schema_extra=_update_json_schema_extra, + ) + + @staticmethod + def compose_resource_name( + parent_name: RelativeResourceName, job_id: UUID + ) -> RelativeResourceName: + # CAREFUL, this is not guarantee a UNIQUE identifier since the resource + # could have some alias entrypoints and the wrong parent_name might be introduced here + collection_or_resource_ids = [ + *split_resource_name(parent_name), + "jobs", + f"{job_id}", + ] + return compose_resource_name(*collection_or_resource_ids) + + +class Job(JobModel): # Get links to other resources url: Annotated[HttpUrl, UriSchema()] | None = Field( ..., description="Link to get this resource (self)" @@ -284,6 +320,11 @@ def _check_name(cls, v, info: ValidationInfo): raise ValueError(msg) return v + @property + def resource_name(self) -> str: + """Relative Resource Name""" + return self.name + # constructors ------ @classmethod @@ -312,24 +353,6 @@ def create_job_from_solver_or_program( inputs_checksum=inputs.compute_checksum(), ) - @classmethod - def compose_resource_name( - cls, parent_name: RelativeResourceName, job_id: UUID - ) -> RelativeResourceName: - # CAREFUL, this is not guarantee a UNIQUE identifier since the resource - # could have some alias entrypoints and the wrong parent_name might be introduced here - collection_or_resource_ids = [ - *split_resource_name(parent_name), - "jobs", - f"{job_id}", - ] - return compose_resource_name(*collection_or_resource_ids) - - @property - def resource_name(self) -> str: - """Relative Resource Name""" - return self.name - def get_url( solver_or_program: Solver | Program, url_for: Callable[..., HttpUrl], job_id: JobID diff --git a/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py index 9c4e67d1f5e..a9488e27234 100644 --- a/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py @@ -15,7 +15,6 @@ from models_library.basic_types import KeyIDStr from models_library.projects import Project from models_library.projects_nodes import InputID -from models_library.rpc.webserver.projects import ProjectRpcGet from pydantic import HttpUrl, TypeAdapter from ..services_http.director_v2 import ComputationTaskGet @@ -183,7 +182,7 @@ def create_new_project_for_job( def create_job_from_project( *, solver_or_program: Solver | Program, - project: ProjectRpcGet | ProjectGet | Project, + project: ProjectGet | Project, url_for: Callable[..., HttpUrl], ) -> Job: """ @@ -198,9 +197,7 @@ def create_job_from_project( assert solver_or_program.version in project.name # nosec assert urllib.parse.quote_plus(solver_or_program.id) in project.name # nosec - # get solver node - node_id = next(iter(project.workbench.keys())) - solver_node: Node = project.workbench[node_id] + solver_node: Node = next(iter(project.workbench.values())) job_inputs: JobInputs = create_job_inputs_from_node_inputs( inputs=solver_node.inputs or {} ) @@ -210,7 +207,7 @@ def create_job_from_project( job_id = project.uuid - job = Job( + return Job( id=job_id, name=project.name, inputs_checksum=job_inputs.compute_checksum(), @@ -225,8 +222,6 @@ def create_job_from_project( ), ) - return job - def create_jobstatus_from_task(task: ComputationTaskGet) -> JobStatus: return JobStatus( From f4e77310f75aa6af6cdd62f8abb2968a09646f68 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 10:49:09 +0200 Subject: [PATCH 16/48] reverted location --- .../src/simcore_service_api_server/_service_job.py | 2 +- .../simcore_service_api_server/_service_solvers.py | 4 +++- .../api/routes/solvers_jobs.py | 6 +++--- .../api/routes/solvers_jobs_getters.py | 2 +- .../api/routes/studies_jobs.py | 2 +- .../simcore_service_api_server/services_http/jobs.py | 2 +- .../solver_job_models_converters.py | 12 ++++++------ ...=> test_services_solver_job_models_converters.py} | 2 +- 8 files changed, 17 insertions(+), 15 deletions(-) rename services/api-server/src/simcore_service_api_server/{models => services_http}/solver_job_models_converters.py (96%) rename services/api-server/tests/unit/{test_models_solver_job_models_converters.py => test_services_solver_job_models_converters.py} (99%) diff --git a/services/api-server/src/simcore_service_api_server/_service_job.py b/services/api-server/src/simcore_service_api_server/_service_job.py index 61d43d8a034..1b7cd79a317 100644 --- a/services/api-server/src/simcore_service_api_server/_service_job.py +++ b/services/api-server/src/simcore_service_api_server/_service_job.py @@ -14,7 +14,7 @@ from .models.schemas.jobs import Job, JobInputs from .models.schemas.programs import Program from .models.schemas.solvers import Solver -from .models.solver_job_models_converters import ( +from .services_http.solver_job_models_converters import ( create_job_from_project, create_new_project_for_job, ) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 952029dfb3d..0b3c2604651 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -19,7 +19,9 @@ from .api.dependencies.webserver_rpc import get_wb_api_rpc_client from .models.schemas.jobs import JobInputs, JobModel from .models.schemas.solvers import Solver, SolverKeyId -from .models.solver_job_models_converters import create_job_inputs_from_node_inputs +from .services_http.solver_job_models_converters import ( + create_job_inputs_from_node_inputs, +) from .services_rpc.catalog import CatalogService from .services_rpc.wb_api_server import WbApiRpcClient diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index f331e4da872..37c97d45e86 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -27,11 +27,11 @@ JobStatus, ) from ...models.schemas.solvers import Solver, SolverKeyId -from ...models.solver_job_models_converters import ( - create_jobstatus_from_task, -) from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import replace_custom_metadata, start_project, stop_project +from ...services_http.solver_job_models_converters import ( + create_jobstatus_from_task, +) from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index a10f7f25108..d98a02b3581 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -41,13 +41,13 @@ WalletGetWithAvailableCreditsLegacy, ) from ...models.schemas.solvers import SolverKeyId -from ...models.solver_job_models_converters import create_job_from_project from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import ( get_custom_metadata, raise_if_job_not_associated_with_solver, ) from ...services_http.log_streaming import LogDistributor, LogStreamer +from ...services_http.solver_job_models_converters import create_job_from_project from ...services_http.solver_job_outputs import ResultsTypes, get_solver_output_results from ...services_http.storage import StorageApi, to_file_api_model from ..dependencies.application import get_reverse_url_mapper diff --git a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py index 8f6edba0cae..0043b5daa70 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/studies_jobs.py @@ -29,7 +29,6 @@ JobStatus, ) from ...models.schemas.studies import JobLogsMap, Study, StudyID -from ...models.solver_job_models_converters import create_jobstatus_from_task from ...services_http.director_v2 import DirectorV2Api from ...services_http.jobs import ( get_custom_metadata, @@ -37,6 +36,7 @@ start_project, stop_project, ) +from ...services_http.solver_job_models_converters import create_jobstatus_from_task from ...services_http.storage import StorageApi from ...services_http.study_job_models_converters import ( create_job_from_study, diff --git a/services/api-server/src/simcore_service_api_server/services_http/jobs.py b/services/api-server/src/simcore_service_api_server/services_http/jobs.py index 5b7b0fd441b..ed2ef50d588 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/jobs.py +++ b/services/api-server/src/simcore_service_api_server/services_http/jobs.py @@ -17,8 +17,8 @@ JobPricingSpecification, JobStatus, ) -from ..models.solver_job_models_converters import create_jobstatus_from_task from .director_v2 import DirectorV2Api +from .solver_job_models_converters import create_jobstatus_from_task from .webserver import AuthSession _logger = logging.getLogger(__name__) diff --git a/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py b/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py similarity index 96% rename from services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py rename to services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py index a9488e27234..43f5d13426d 100644 --- a/services/api-server/src/simcore_service_api_server/models/solver_job_models_converters.py +++ b/services/api-server/src/simcore_service_api_server/services_http/solver_job_models_converters.py @@ -17,10 +17,9 @@ from models_library.projects_nodes import InputID from pydantic import HttpUrl, TypeAdapter -from ..services_http.director_v2 import ComputationTaskGet -from .domain.projects import InputTypes, Node, SimCoreFileLink -from .schemas.files import File -from .schemas.jobs import ( +from ..models.domain.projects import InputTypes, Node, SimCoreFileLink +from ..models.schemas.files import File +from ..models.schemas.jobs import ( ArgumentTypes, Job, JobInputs, @@ -30,8 +29,9 @@ get_runner_url, get_url, ) -from .schemas.programs import Program -from .schemas.solvers import Solver +from ..models.schemas.programs import Program +from ..models.schemas.solvers import Solver +from .director_v2 import ComputationTaskGet # UTILS ------ _BASE_UUID = uuid.UUID("231e13db-6bc6-4f64-ba56-2ee2c73b9f09") diff --git a/services/api-server/tests/unit/test_models_solver_job_models_converters.py b/services/api-server/tests/unit/test_services_solver_job_models_converters.py similarity index 99% rename from services/api-server/tests/unit/test_models_solver_job_models_converters.py rename to services/api-server/tests/unit/test_services_solver_job_models_converters.py index 3aba0cf9ecf..1e926f09c86 100644 --- a/services/api-server/tests/unit/test_models_solver_job_models_converters.py +++ b/services/api-server/tests/unit/test_services_solver_job_models_converters.py @@ -10,7 +10,7 @@ from simcore_service_api_server.models.schemas.files import File from simcore_service_api_server.models.schemas.jobs import ArgumentTypes, Job, JobInputs from simcore_service_api_server.models.schemas.solvers import Solver -from simcore_service_api_server.models.solver_job_models_converters import ( +from simcore_service_api_server.services_http.solver_job_models_converters import ( create_job_from_project, create_job_inputs_from_node_inputs, create_jobstatus_from_task, From 281eeecf2fd95011a01cd57481e909feea4264a8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 10:57:05 +0200 Subject: [PATCH 17/48] =?UTF-8?q?=E2=9C=A8=20Rename=20ProjectJobRpcGet=20t?= =?UTF-8?q?o=20ProjectAsJobRpcGet=20for=20consistency=20across=20the=20cod?= =?UTF-8?q?ebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models-library/src/models_library/rpc/webserver/projects.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index ab0a16fe0d7..8b4ac6b44ae 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -54,6 +54,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } ) + # Specific to jobs job_parent_resource_name: str @staticmethod From e6093552f4135232b61f5df32a9a900e84e68a92 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:27:31 +0200 Subject: [PATCH 18/48] =?UTF-8?q?=E2=9C=A8=20Update=20ProjectJobRpcGet=20a?= =?UTF-8?q?nd=20ProjectJobDBGet=20models=20to=20include=20workbench=20stru?= =?UTF-8?q?cture=20and=20fix=20schema=20update=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models_library/rpc/webserver/projects.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 8b4ac6b44ae..4a5805fcfda 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -54,26 +54,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: } ) - # Specific to jobs - job_parent_resource_name: str - - @staticmethod - def _update_json_schema_extra(schema: JsonDict) -> None: - schema.update( - { - "examples": [ - { - "uuid": "12345678-1234-5678-1234-123456789012", - "name": "My project", - "description": "My project description", - "creation_date": "2023-01-01T00:00:00Z", - "last_change_date": "2023-01-01T00:00:00Z", - "job_parent_resource_name": "solvers/foo/release/1.2.3", - "workbench": {}, - }, - ] - } - ) model_config = ConfigDict( extra="forbid", From c577b21a140cc30aedde9d844dbc4ec1357c4432 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:25:19 +0200 Subject: [PATCH 19/48] =?UTF-8?q?=E2=9C=A8=20Fix=20timestamp=20field=20nam?= =?UTF-8?q?e=20from=20'creation=5Fat'=20to=20'created=5Fat'=20in=20Project?= =?UTF-8?q?JobRpcGet=20model=20and=20related=20usages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models_library/rpc/webserver/projects.py | 7 +-- .../helpers/webserver_rpc_server.py | 9 ++- .../_service_solvers.py | 31 +++++---- .../models/schemas/jobs.py | 63 +++++++------------ .../services_rpc/wb_api_server.py | 1 - services/api-server/tests/unit/conftest.py | 4 +- .../tests/unit/test_service_solvers.py | 27 +------- .../projects/_controller/projects_rpc.py | 2 +- 8 files changed, 54 insertions(+), 90 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 4a5805fcfda..21ca6a8310d 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -29,7 +29,7 @@ class ProjectJobRpcGet(BaseModel): workbench: NodesDict # timestamps - creation_at: datetime + created_at: datetime modified_at: datetime # Specific to jobs @@ -46,15 +46,14 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "name": "My project", "description": "My project description", "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, - "creation_at": "2023-01-01T00:00:00Z", + "created_at": "2023-01-01T00:00:00Z", "modified_at": "2023-01-01T00:00:00Z", - "job_parent_resource_name": "solvers/foo/release/1.2.3", + "job_parent_resource_name": "solvers/slv_123/release/1.2.3", }, ] } ) - model_config = ConfigDict( extra="forbid", populate_by_name=True, diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index 5bdbb2336b2..bb1f3558c9b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -8,7 +8,10 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.rest_pagination import PageOffsetInt -from models_library.rpc.webserver.projects import PageRpcProjectJobRpcGet +from models_library.rpc.webserver.projects import ( + PageRpcProjectJobRpcGet, + ProjectJobRpcGet, +) from models_library.rpc_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt, @@ -63,10 +66,10 @@ async def list_projects_marked_as_jobs( if job_parent_resource_name_filter: assert not job_parent_resource_name_filter.startswith("/") - items = PageRpcProjectJobRpcGet.model_json_schema()["examples"] + items = ProjectJobRpcGet.model_json_schema()["examples"] return PageRpcProjectJobRpcGet.create( - items[offset, : offset + limit], + items[offset : offset + limit], total=len(items), limit=limit, offset=offset, diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 0b3c2604651..a12aaaea72d 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -17,7 +17,7 @@ from packaging.version import Version from .api.dependencies.webserver_rpc import get_wb_api_rpc_client -from .models.schemas.jobs import JobInputs, JobModel +from .models.schemas.jobs import Job, JobInputs from .models.schemas.solvers import Solver, SolverKeyId from .services_http.solver_job_models_converters import ( create_job_inputs_from_node_inputs, @@ -97,7 +97,7 @@ async def list_jobs( product_name: ProductName, offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, - ) -> tuple[list[JobModel], PageMetaInfoLimitOffset]: + ) -> tuple[list[Job], PageMetaInfoLimitOffset]: """Lists all solver jobs for a user with pagination""" # NOTE: perhaps we should get comp_tasks instead of projects! or a combinatino of both? @@ -111,27 +111,32 @@ async def list_jobs( job_parent_resource_name_filter="solvers", # TODO: project shouldr eturn parent resource name and workbench ) - jobs: list[JobModel] = [] + jobs: list[Job] = [] - for prj in projects_page.data: + for project_job in projects_page.data: - assert len(prj.workbench) == 1, "Expected only one solver node in workbench" + assert ( + len(project_job.workbench) == 1 + ), "Expected only one solver node in workbench" - solver_node: Node = next(iter(prj.workbench.values())) + solver_node: Node = next(iter(project_job.workbench.values())) job_inputs: JobInputs = create_job_inputs_from_node_inputs( inputs=solver_node.inputs or {} ) - assert prj.job_parent_resource_name # nosec + assert project_job.job_parent_resource_name # nosec jobs.append( - JobModel( - id=prj.uuid, - name=JobModel.compose_resource_name( - prj.job_parent_resource_name, prj.uuid + Job( + id=project_job.uuid, + name=Job.compose_resource_name( + project_job.job_parent_resource_name, project_job.uuid ), inputs_checksum=job_inputs.compute_checksum(), - created_at=prj.creation_date, - runner_name=prj.job_parent_resource_name, + created_at=project_job.created_at, + runner_name=project_job.job_parent_resource_name, + url=None, + runner_url=None, + outputs_url=None, ) ) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index 9e3e11c2a4d..4daf3035e12 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -27,7 +27,6 @@ ValidationInfo, field_validator, ) -from pydantic.types import JsonDict from servicelib.logging_utils import LogLevelInt, LogMessageStr from starlette.datastructures import Headers @@ -238,7 +237,7 @@ class JobMetadata(BaseModel): # "jobs/c2789bd2-7385-4e00-91d3-2f100df41185" -class JobModel(BaseModel): +class Job(BaseModel): id: JobID name: RelativeResourceName @@ -250,41 +249,6 @@ class JobModel(BaseModel): ..., description="Runner that executes job" ) - @staticmethod - def _update_json_schema_extra(schema: JsonDict) -> None: - schema.udpate( - { - "examples": [ - { - "id": "f622946d-fd29-35b9-a193-abdd1095167c", - "name": "solvers/isolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c", - "runner_name": "solvers/isolve/releases/1.3.4", - "inputs_checksum": "12345", - "created_at": "2021-01-22T23:59:52.322176", - }, - ] - } - ) - - model_config = ConfigDict( - json_schema_extra=_update_json_schema_extra, - ) - - @staticmethod - def compose_resource_name( - parent_name: RelativeResourceName, job_id: UUID - ) -> RelativeResourceName: - # CAREFUL, this is not guarantee a UNIQUE identifier since the resource - # could have some alias entrypoints and the wrong parent_name might be introduced here - collection_or_resource_ids = [ - *split_resource_name(parent_name), - "jobs", - f"{job_id}", - ] - return compose_resource_name(*collection_or_resource_ids) - - -class Job(JobModel): # Get links to other resources url: Annotated[HttpUrl, UriSchema()] | None = Field( ..., description="Link to get this resource (self)" @@ -320,11 +284,6 @@ def _check_name(cls, v, info: ValidationInfo): raise ValueError(msg) return v - @property - def resource_name(self) -> str: - """Relative Resource Name""" - return self.name - # constructors ------ @classmethod @@ -353,6 +312,26 @@ def create_job_from_solver_or_program( inputs_checksum=inputs.compute_checksum(), ) + @classmethod + def compose_resource_name( + cls, parent_name: RelativeResourceName, job_id: UUID + ) -> RelativeResourceName: + assert "jobs" not in parent_name # nosec + + # CAREFUL, this is not guarantee a UNIQUE identifier since the resource + # could have some alias entrypoints and the wrong parent_name might be introduced here + collection_or_resource_ids = [ + *split_resource_name(parent_name), + "jobs", + f"{job_id}", + ] + return compose_resource_name(*collection_or_resource_ids) + + @property + def resource_name(self) -> str: + """Relative Resource Name""" + return self.name + def get_url( solver_or_program: Solver | Program, url_for: Callable[..., HttpUrl], job_id: JobID diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index 670cc03952c..fa93aeb3519 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -219,7 +219,6 @@ async def mark_project_as_job( job_parent_resource_name=job_parent_resource_name, ) - @_exception_mapper(rpc_exception_map={}) async def list_projects_marked_as_jobs( self, *, diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 704bbc84f57..7856e38da42 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -354,12 +354,12 @@ def mocked_webserver_rpc_api( "mark_project_as_job": mocker.patch.object( projects_rpc, "mark_project_as_job", - side_effects.mark_project_as_job, + side_effect=side_effects.mark_project_as_job, ), "list_projects_marked_as_jobs": mocker.patch.object( projects_rpc, "list_projects_marked_as_jobs", - side_effects.list_projects_marked_as_jobs, + side_effect=side_effects.list_projects_marked_as_jobs, ), } diff --git a/services/api-server/tests/unit/test_service_solvers.py b/services/api-server/tests/unit/test_service_solvers.py index 2c2f5727649..35655fdfd3d 100644 --- a/services/api-server/tests/unit/test_service_solvers.py +++ b/services/api-server/tests/unit/test_service_solvers.py @@ -54,29 +54,8 @@ async def test_list_jobs( product_name=product_name, ) assert isinstance(jobs, list) - mocked_webserver_rpc_api["list_projects_marked_as_jobs"].assert_called_once_with( - product_name=product_name, - user_id=user_id, - offset=0, - limit=999, - job_parent_resource_name_filter="solvers", - ) + mocked_webserver_rpc_api["list_projects_marked_as_jobs"].assert_called_once() assert page_meta.total >= 0 - assert page_meta.limit == 999 + assert page_meta.limit == 49 assert page_meta.offset == 0 - - # Test with explicit pagination - mocked_webserver_rpc_api["list_projects_marked_as_jobs"].reset_mock() - jobs, page_meta = await solver_service.list_jobs( - user_id=user_id, - product_name=product_name, - offset=10, - limit=20, - ) - mocked_webserver_rpc_api["list_projects_marked_as_jobs"].assert_called_once_with( - product_name=product_name, - user_id=user_id, - offset=10, - limit=20, - job_parent_resource_name_filter="solvers", - ) + assert page_meta.count > 0 diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index d3bd67d0bbf..1e73b3ab50c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -84,7 +84,7 @@ async def list_projects_marked_as_jobs( name=project.name, description=project.description, workbench=project.workbench, - creation_at=project.creation_date, + created_at=project.creation_date, modified_at=project.last_change_date, job_parent_resource_name=project.job_parent_resource_name, ) From 4fdff6e99ea981ee8f0af28b5dca5dd57085a1de Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 20:08:43 +0200 Subject: [PATCH 20/48] =?UTF-8?q?=E2=9C=A8=20Add=20string=20trimming=20val?= =?UTF-8?q?idator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models_library/utils/common_validators.py | 13 ++++++++++-- .../models/schemas/_base.py | 20 ++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/utils/common_validators.py b/packages/models-library/src/models_library/utils/common_validators.py index 7caf338e61b..c55db09c5f5 100644 --- a/packages/models-library/src/models_library/utils/common_validators.py +++ b/packages/models-library/src/models_library/utils/common_validators.py @@ -1,4 +1,4 @@ -""" Reusable validators +"""Reusable validators Example: @@ -22,10 +22,19 @@ class MyModel(BaseModel): from common_library.json_serialization import json_loads from orjson import JSONDecodeError -from pydantic import BaseModel +from pydantic import BaseModel, BeforeValidator from pydantic.alias_generators import to_camel +def trim_string_before(max_length: int) -> BeforeValidator: + def _trim(value: str): + if isinstance(value, str): + return value[:max_length] + return value + + return BeforeValidator(_trim) + + def empty_str_to_none_pre_validator(value: Any): if isinstance(value, str) and value.strip() == "": return None diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py index 30d9ca3ba31..187de8ce20f 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py @@ -3,6 +3,7 @@ import packaging.version from models_library.utils.change_case import camel_to_snake +from models_library.utils.common_validators import trim_string_before from pydantic import BaseModel, ConfigDict, Field, HttpUrl, StringConstraints from ...models._utils_pydantic import UriSchema @@ -28,22 +29,31 @@ class ApiServerInputSchema(BaseModel): class BaseService(BaseModel): - id: Annotated[str, Field(..., description="Resource identifier")] + id: Annotated[ + str, + Field(description="Resource identifier"), + ] version: Annotated[ - VersionStr, Field(..., description="Semantic version number of the resource") + VersionStr, + Field(description="Semantic version number of the resource"), ] title: Annotated[ str, + trim_string_before(max_length=100), StringConstraints(max_length=100), - Field(..., description="Human readable name"), + Field(description="Human readable name"), ] description: Annotated[ str | None, - StringConstraints(max_length=500), + trim_string_before(max_length=1000), + StringConstraints(max_length=1000), Field(default=None, description="Description of the resource"), ] + url: Annotated[ - HttpUrl | None, UriSchema(), Field(..., description="Link to get this resource") + HttpUrl | None, + UriSchema(), + Field(description="Link to get this resource"), ] @property From b3fffd9d57251a138609644fee160ca882b981cf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 24 Apr 2025 20:09:20 +0200 Subject: [PATCH 21/48] =?UTF-8?q?=E2=9C=A8=20Enhance=20job=20listing=20fun?= =?UTF-8?q?ctionality=20by=20adding=20solver=20filters=20and=20updating=20?= =?UTF-8?q?job=20URLs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 17 ++++++++-- .../api/routes/solvers_jobs_getters.py | 21 ++++++++++-- .../models/schemas/jobs.py | 32 ++++++++++++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index a12aaaea72d..268c29d7d1d 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -95,6 +95,10 @@ async def list_jobs( *, user_id: UserID, product_name: ProductName, + # filters + solver_id: SolverKeyId | None = None, + solver_version: VersionStr | None = None, + # pagination offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: @@ -102,20 +106,29 @@ async def list_jobs( # NOTE: perhaps we should get comp_tasks instead of projects! or a combinatino of both? # I need inputs_checksum and job_parent_source_name! + # FIXME: this is still buggy. soldve_id and solver_version need to be encoded! + job_parent_resource_name_filter = "solvers" + if solver_id: + job_parent_resource_name_filter += f"/{solver_id}" + if solver_version: + job_parent_resource_name_filter += f"/{solver_version}" + elif solver_version: + msg = "solver_version is set but solver_id is not. Please provide both or none of them" + raise ValueError(msg) projects_page = await self._webserver_client.list_projects_marked_as_jobs( product_name=product_name, user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter="solvers", # TODO: project shouldr eturn parent resource name and workbench + job_parent_resource_name_filter=job_parent_resource_name_filter, ) jobs: list[Job] = [] for project_job in projects_page.data: - assert ( + assert ( # nosec len(project_job.workbench) == 1 ), "Expected only one solver node in workbench" diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index d98a02b3581..f8f813a710b 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -18,6 +18,7 @@ from pydantic.types import PositiveInt from servicelib.fastapi.requests_decorators import cancel_on_disconnect from servicelib.logging_utils import log_context +from simcore_service_api_server.models.api_resources import parse_resources_ids from sqlalchemy.ext.asyncio import AsyncEngine from ..._service_solvers import SolverService @@ -35,6 +36,7 @@ JobLog, JobMetadata, JobOutputs, + update_job_urls, ) from ...models.schemas.model_adapter import ( PricingUnitGetLegacy, @@ -131,11 +133,26 @@ async def list_all_solvers_jobs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], page_params: Annotated[PaginationParams, Depends()], solver_service: Annotated[SolverService, Depends(SolverService)], - webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): - raise NotImplementedError + + jobs, meta = await solver_service.list_jobs( + product_name=product_name, + user_id=user_id, + offset=page_params.offset, + limit=page_params.limit, + ) + + for job in jobs: + solver_key, version, job_id = parse_resources_ids(job.resource_name) + update_job_urls(job, solver_key, version, job_id, url_for) + + return create_page( + jobs, + total=meta.total, + params=page_params, + ) @router.get( diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index 4daf3035e12..9094f4a6b27 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -50,7 +50,7 @@ # - custom metadata # from .programs import Program, ProgramKeyId -from .solvers import Solver +from .solvers import Solver, SolverKeyId JobID: TypeAlias = UUID @@ -371,6 +371,36 @@ def get_outputs_url( return None +def update_job_urls( + job: Job, + solver_key: SolverKeyId, + solver_version: VersionStr, + job_id: JobID | str, + url_for: Callable[..., HttpUrl], +) -> Job: + job.url = url_for( + "get_job", + solver_key=solver_key, + version=solver_version, + job_id=job_id, + ) + + job.runner_url = url_for( + "get_solver_release", + solver_key=solver_key, + version=solver_version, + ) + + job.outputs_url = url_for( + "get_job_outputs", + solver_key=solver_key, + version=solver_version, + job_id=job_id, + ) + + return job + + PercentageInt: TypeAlias = Annotated[int, Field(ge=0, le=100)] From 0cc3872652c5a108728b17c9c9071c1364ecd00f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:12:49 +0200 Subject: [PATCH 22/48] doc --- .../models/api_resources.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 1f3e4e71e38..2ae99972e51 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -58,21 +58,31 @@ def split_resource_name(resource_name: RelativeResourceName) -> list[str]: return [f"{urllib.parse.unquote_plus(p)}" for p in quoted_parts] -def split_resource_name_as_dict( - resource_name: RelativeResourceName, -) -> dict[str, str | None]: - """Returns a map with - resource_ids[Collection-ID] == Resource-ID - """ - parts = split_resource_name(resource_name) - return dict(zip(parts[::2], parts[1::2], strict=False)) - - def parse_collections_ids(resource_name: RelativeResourceName) -> list[str]: + """ + Example: + resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" + returns ["solvers", "releases", "jobs", "outputs"] + """ parts = split_resource_name(resource_name) return parts[::2] def parse_resources_ids(resource_name: RelativeResourceName) -> list[str]: + """ + Example: + resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" + returns ["simcore/services/comp/isolve", "1.3.4", "f622946d-fd29-35b9-a193-abdd1095167c", "output 22"] + """ parts = split_resource_name(resource_name) return parts[1::2] + + +def split_resource_name_as_dict( + resource_name: RelativeResourceName, +) -> dict[str, str | None]: + """ + Returns a map such as resource_ids[Collection-ID] == Resource-ID + """ + parts = split_resource_name(resource_name) + return dict(zip(parts[::2], parts[1::2], strict=False)) From ba4e1ebe1564d4615e8290a1cc015cc456f31a95 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 15:24:24 +0200 Subject: [PATCH 23/48] =?UTF-8?q?=E2=9C=A8=20Rename=20job=5Fparent=5Fresou?= =?UTF-8?q?rce=5Fname=5Ffilter=20to=20job=5Fparent=5Fresource=5Fname=5Fpre?= =?UTF-8?q?fix=20for=20consistency=20across=20the=20codebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pytest_simcore/helpers/webserver_rpc_server.py | 8 +++++--- .../rabbitmq/rpc_interfaces/webserver/projects.py | 4 ++-- .../services_rpc/wb_api_server.py | 4 ++-- .../projects/_controller/projects_rpc.py | 4 ++-- .../projects/_jobs_repository.py | 9 +++++---- .../simcore_service_webserver/projects/_jobs_service.py | 4 ++-- .../unit/with_dbs/02/test_projects__jobs_service.py | 6 +++--- .../server/tests/unit/with_dbs/02/test_projects_rpc.py | 2 +- 8 files changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index bb1f3558c9b..13617f25ab1 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -57,14 +57,16 @@ async def list_projects_marked_as_jobs( offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, # filters - job_parent_resource_name_filter: str | None = None, + job_parent_resource_name_prefix: str | None = None, ) -> PageRpcProjectJobRpcGet: assert rpc_client assert product_name assert user_id - if job_parent_resource_name_filter: - assert not job_parent_resource_name_filter.startswith("/") + if job_parent_resource_name_prefix: + assert not job_parent_resource_name_prefix.startswith("/") + assert not job_parent_resource_name_prefix.endswith("%") + assert not job_parent_resource_name_prefix.startswith("%") items = ProjectJobRpcGet.model_json_schema()["examples"] diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py index ccfaff4028c..f2e261b7d6a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/webserver/projects.py @@ -52,7 +52,7 @@ async def list_projects_marked_as_jobs( offset: PageOffsetInt = 0, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, # filters - job_parent_resource_name_filter: str | None = None, + job_parent_resource_name_prefix: str | None = None, ) -> PageRpcProjectJobRpcGet: result = await rpc_client.request( WEBSERVER_RPC_NAMESPACE, @@ -61,7 +61,7 @@ async def list_projects_marked_as_jobs( user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter=job_parent_resource_name_filter, + job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) assert TypeAdapter(PageRpcProjectJobRpcGet).validate_python(result) # nosec return cast(PageRpcProjectJobRpcGet, result) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index fa93aeb3519..495e72cd08a 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -226,7 +226,7 @@ async def list_projects_marked_as_jobs( user_id: UserID, offset: int = 0, limit: int = 50, - job_parent_resource_name_filter: str | None = None, + job_parent_resource_name_prefix: str | None = None, ): return await projects_rpc.list_projects_marked_as_jobs( rpc_client=self._client, @@ -234,7 +234,7 @@ async def list_projects_marked_as_jobs( user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter=job_parent_resource_name_filter, + job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py index 1e73b3ab50c..6ee4861b690 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rpc.py @@ -66,7 +66,7 @@ async def list_projects_marked_as_jobs( offset: PageOffsetInt, limit: PageLimitInt, # filters - job_parent_resource_name_filter: str | None, + job_parent_resource_name_prefix: str | None, ) -> PageRpcProjectJobRpcGet: total, projects = await _jobs_service.list_my_projects_marked_as_jobs( @@ -75,7 +75,7 @@ async def list_projects_marked_as_jobs( user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter=job_parent_resource_name_filter, + job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) job_projects = [ diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 5981f5fe7b0..04ac1a34dec 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -62,12 +62,13 @@ async def list_projects_marked_as_jobs( user_id: UserID, offset: int = 0, limit: int = 10, - job_parent_resource_name_filter: str | None = None, + job_parent_resource_name_prefix: str | None = None, ) -> tuple[int, list[ProjectJobDBGet]]: """ Lists projects marked as jobs for a specific user and product - """ + Example: job_parent_resource_name = "/solvers/solver1" + """ # Step 1: Get group IDs associated with the user user_groups_query = ( sa.select(user_to_groups.c.gid) @@ -96,10 +97,10 @@ async def list_projects_marked_as_jobs( ) # Apply job_parent_resource_name_filter if provided - if job_parent_resource_name_filter: + if job_parent_resource_name_prefix: access_query = access_query.where( projects_to_jobs.c.job_parent_resource_name.like( - f"%{job_parent_resource_name_filter}%" + f"{job_parent_resource_name_prefix}%" ) ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py index 712549bdc6c..7b4289644c2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_service.py @@ -56,7 +56,7 @@ async def list_my_projects_marked_as_jobs( user_id: UserID, offset: int = 0, limit: int = 10, - job_parent_resource_name_filter: str | None = None, + job_parent_resource_name_prefix: str | None = None, ) -> tuple[int, list[ProjectJobDBGet]]: """ Lists paginated projects marked as jobs for the given user and product. @@ -68,5 +68,5 @@ async def list_my_projects_marked_as_jobs( product_name=product_name, offset=offset, limit=limit, - job_parent_resource_name_filter=job_parent_resource_name_filter, + job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py b/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py index c33274ca609..ebe796ad9f0 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects__jobs_service.py @@ -158,7 +158,7 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_filter=project_job_fixture.job_parent_resource_name, + job_parent_resource_name_prefix=project_job_fixture.job_parent_resource_name, ) assert total_count == 1 assert len(result) == 1 @@ -174,7 +174,7 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_filter="test/%", + job_parent_resource_name_prefix="test/%", ) assert total_count == 1 assert len(result) == 1 @@ -190,7 +190,7 @@ async def test_user_can_filter_marked_project( app=client.app, product_name=osparc_product_name, user_id=project_job_fixture.user_id, - job_parent_resource_name_filter="other/%", + job_parent_resource_name_prefix="other/%", ) assert total_count == 0 assert len(result) == 0 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py index b18067e5383..28147c93d3e 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_rpc.py @@ -114,7 +114,7 @@ async def test_rpc_client_list_my_projects_marked_as_jobs( rpc_client=rpc_client, product_name=product_name, user_id=user_id, - job_parent_resource_name_filter="solvers/solver123", + job_parent_resource_name_prefix="solvers/solver123", ) assert page.meta.total == 1 From ca09f43cc2f2b355187cfea9f0dac1c2d2cca229 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 17:50:09 +0200 Subject: [PATCH 24/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20job=20parent=20res?= =?UTF-8?q?ource=20name=20handling=20in=20list=5Fjobs=20method=20for=20imp?= =?UTF-8?q?roved=20clarity=20and=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 268c29d7d1d..4d1fa8f36cb 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -17,6 +17,7 @@ from packaging.version import Version from .api.dependencies.webserver_rpc import get_wb_api_rpc_client +from .models.api_resources import compose_resource_name from .models.schemas.jobs import Job, JobInputs from .models.schemas.solvers import Solver, SolverKeyId from .services_http.solver_job_models_converters import ( @@ -104,28 +105,34 @@ async def list_jobs( ) -> tuple[list[Job], PageMetaInfoLimitOffset]: """Lists all solver jobs for a user with pagination""" - # NOTE: perhaps we should get comp_tasks instead of projects! or a combinatino of both? - # I need inputs_checksum and job_parent_source_name! - # FIXME: this is still buggy. soldve_id and solver_version need to be encoded! - job_parent_resource_name_filter = "solvers" + # 1. Compose job parent resource name prefix + collection_or_resource_ids = [ + "solvers", # solver_id, "releases", solver_version, "jobs", + ] if solver_id: - job_parent_resource_name_filter += f"/{solver_id}" + collection_or_resource_ids.append(solver_id) if solver_version: - job_parent_resource_name_filter += f"/{solver_version}" + collection_or_resource_ids.append("releases") + collection_or_resource_ids.append(solver_version) elif solver_version: msg = "solver_version is set but solver_id is not. Please provide both or none of them" raise ValueError(msg) + job_parent_resource_name_prefix = compose_resource_name( + *collection_or_resource_ids + ) + + # 2. List projects marked as jobs projects_page = await self._webserver_client.list_projects_marked_as_jobs( product_name=product_name, user_id=user_id, offset=offset, limit=limit, - job_parent_resource_name_filter=job_parent_resource_name_filter, + job_parent_resource_name_prefix=job_parent_resource_name_prefix, ) + # 3. Convert projects to jobs jobs: list[Job] = [] - for project_job in projects_page.data: assert ( # nosec From f50c33fa469688969cf23f4d2dddda8ed040f13b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 21:11:48 +0200 Subject: [PATCH 25/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20service=20dependen?= =?UTF-8?q?cy=20injection=20to=20use=20get=5Fsolver=5Fservice=20for=20impr?= =?UTF-8?q?oved=20clarity=20and=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/dependencies/services.py | 37 +++++++++++++++++-- .../api/routes/solvers_jobs_getters.py | 10 ++--- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/services.py b/services/api-server/src/simcore_service_api_server/api/dependencies/services.py index 5db67660bc5..33eedc5379f 100644 --- a/services/api-server/src/simcore_service_api_server/api/dependencies/services.py +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/services.py @@ -1,11 +1,15 @@ -""" Dependences with any other services (except webserver) - -""" from collections.abc import Callable +from typing import Annotated -from fastapi import HTTPException, Request, status +from fastapi import Depends, HTTPException, Request, status +from servicelib.rabbitmq import RabbitMQRPCClient +from ..._service_solvers import SolverService +from ...services_rpc.catalog import CatalogService +from ...services_rpc.wb_api_server import WbApiRpcClient from ...utils.client_base import BaseServiceClientApi +from .rabbitmq import get_rabbitmq_rpc_client +from .webserver_rpc import get_wb_api_rpc_client def get_api_client(client_type: type[BaseServiceClientApi]) -> Callable: @@ -32,3 +36,28 @@ def _get_client_from_app(request: Request) -> BaseServiceClientApi: return client_obj return _get_client_from_app + + +def get_catalog_service( + rpc_client: Annotated[RabbitMQRPCClient, Depends(get_rabbitmq_rpc_client)], +): + """ + "Assembles" the CatalogService layer to the RabbitMQ client + in the context of the rest controller (i.e. api/dependencies) + """ + return CatalogService(client=rpc_client) + + +def get_solver_service( + catalog_service: Annotated[CatalogService, Depends(get_catalog_service)], + webserver_client: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], +) -> SolverService: + """ + "Assembles" the SolverService layer to the underlying service and client interfaces + in the context of the rest controller (i.e. api/dependencies) + """ + + return SolverService( + catalog_service=catalog_service, + webserver_client=webserver_client, + ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index f8f813a710b..df19d68512d 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -56,7 +56,7 @@ from ..dependencies.authentication import get_current_user_id, get_product_name from ..dependencies.database import get_db_asyncpg_engine from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor -from ..dependencies.services import get_api_client +from ..dependencies.services import get_api_client, get_solver_service from ..dependencies.webserver_http import AuthSession, get_webserver_session from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import ( @@ -132,7 +132,7 @@ async def list_all_solvers_jobs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], page_params: Annotated[PaginationParams, Depends()], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): @@ -176,7 +176,7 @@ async def list_jobs( solver_key: SolverKeyId, version: VersionStr, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], @@ -222,7 +222,7 @@ async def get_jobs_page( version: VersionStr, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], page_params: Annotated[PaginationParams, Depends()], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], @@ -266,7 +266,7 @@ async def get_job( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], webserver_api: Annotated[AuthSession, Depends(get_webserver_session)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): """Gets job of a given solver""" From c22afb94b61e381b89a143fb645e1dc3c7b1f07b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 21:31:36 +0200 Subject: [PATCH 26/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20test=20fixtures=20?= =?UTF-8?q?for=20improved=20clarity=20and=20consistency=20in=20mocking=20A?= =?UTF-8?q?PIs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/unit/_with_db/conftest.py | 1 + .../tests/unit/_with_db/test_product.py | 31 +++++---- services/api-server/tests/unit/conftest.py | 64 +++++++++---------- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/services/api-server/tests/unit/_with_db/conftest.py b/services/api-server/tests/unit/_with_db/conftest.py index bc93e0442c0..fd2441c879e 100644 --- a/services/api-server/tests/unit/_with_db/conftest.py +++ b/services/api-server/tests/unit/_with_db/conftest.py @@ -272,6 +272,7 @@ async def create_fake_api_keys( create_user_ids: Callable[[PositiveInt], AsyncGenerator[PositiveInt, None]], create_product_names: Callable[[PositiveInt], AsyncGenerator[str, None]], ) -> AsyncGenerator[Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], None]: + async def _generate_fake_api_key(n: PositiveInt): users, products = create_user_ids(n), create_product_names(n) excluded_column = "api_secret" diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index 0bca5b02801..775511713a2 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -29,22 +29,22 @@ async def test_product_webserver( mocked_webserver_rest_api_base: respx.MockRouter, create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], faker: Faker, -) -> None: +): assert client - keys: dict[int, ApiKeyInDB] = {} + wallet_to_api_keys_map: dict[int, ApiKeyInDB] = {} wallet_id: int = faker.pyint(min_value=1) - async for key in create_fake_api_keys(2): + async for api_key in create_fake_api_keys(2): wallet_id += faker.pyint(min_value=1) - keys[wallet_id] = key + wallet_to_api_keys_map[wallet_id] = api_key def _check_key_product_compatibility(request: httpx.Request, **kwargs): assert ( received_product_name := request.headers.get("x-simcore-products-name") ) is not None assert (wallet_id := kwargs.get("wallet_id")) is not None - assert (key := keys[int(wallet_id)]) is not None - assert key.product_name == received_product_name + assert (api_key := wallet_to_api_keys_map[int(wallet_id)]) is not None + assert api_key.product_name == received_product_name return httpx.Response( status.HTTP_200_OK, json=jsonable_encoder( @@ -53,7 +53,7 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): wallet_id=wallet_id, name="my_wallet", description="this is my wallet", - owner=key.id_, + owner=api_key.id_, thumbnail="something", status=WalletStatus.ACTIVE, created=datetime.datetime.now(), @@ -68,30 +68,29 @@ def _check_key_product_compatibility(request: httpx.Request, **kwargs): path__regex=r"/wallets/(?P[-+]?\d+)" ).mock(side_effect=_check_key_product_compatibility) - for wallet_id in keys: - key = keys[wallet_id] + for wallet_id, api_key in wallet_to_api_keys_map.items(): response = await client.get( f"{API_VTAG}/wallets/{wallet_id}", - auth=httpx.BasicAuth(key.api_key, key.api_secret), + auth=httpx.BasicAuth(api_key.api_key, api_key.api_secret), ) assert response.status_code == status.HTTP_200_OK - assert wallet_get_mock.call_count == len(keys) + assert wallet_get_mock.call_count == len(wallet_to_api_keys_map) async def test_product_catalog( client: httpx.AsyncClient, mocked_catalog_rpc_api: dict[str, MockType], create_fake_api_keys: Callable[[PositiveInt], AsyncGenerator[ApiKeyInDB, None]], -) -> None: +): assert client - keys: list[ApiKeyInDB] = [key async for key in create_fake_api_keys(2)] - assert len({key.product_name for key in keys}) == 2 + valid_api_auths: list[ApiKeyInDB] = [key async for key in create_fake_api_keys(2)] + assert len({key.product_name for key in valid_api_auths}) == 2 - for key in keys: + for api_auth in valid_api_auths: await client.get( f"{API_VTAG}/solvers/simcore/services/comp/isolve/releases/2.0.24", - auth=httpx.BasicAuth(key.api_key, key.api_secret), + auth=httpx.BasicAuth(api_auth.api_key, api_auth.api_secret), ) assert mocked_catalog_rpc_api["get_service"].called diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 7856e38da42..9a616413a4c 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -2,10 +2,11 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable +# pylint: disable=broad-exception-caught import json import subprocess -from collections.abc import AsyncIterator, Callable, Iterable, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from copy import deepcopy from pathlib import Path from typing import Any @@ -45,9 +46,6 @@ from pytest_simcore.simcore_webserver_projects_rest_api import GET_PROJECT from requests.auth import HTTPBasicAuth from respx import MockRouter -from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc -from simcore_service_api_server.api.dependencies.rabbitmq import get_rabbitmq_rpc_client from simcore_service_api_server.core.application import init_app from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.repository.api_keys import UserAndProductTuple @@ -92,10 +90,19 @@ def app_environment( def mock_missing_plugins(app_environment: EnvVarsDict, mocker: MockerFixture): settings = ApplicationSettings.create_from_envs() if settings.API_SERVER_RABBITMQ is None: - mocker.patch("simcore_service_api_server.core.application.setup_rabbitmq") - mocker.patch( - "simcore_service_api_server.core._prometheus_instrumentation.setup_prometheus_instrumentation" + import simcore_service_api_server.core.application + + mocker.patch.object( + simcore_service_api_server.core.application, + "setup_rabbitmq", + autospec=True, ) + mocker.patch.object( + simcore_service_api_server.core.application, + "setup_prometheus_instrumentation", + autospec=True, + ) + return app_environment @@ -331,22 +338,13 @@ def mocked_webserver_rest_api_base( @pytest.fixture -def mocked_webserver_rpc_api( - app: FastAPI, mocker: MockerFixture -) -> dict[str, MockType]: - from servicelib.rabbitmq.rpc_interfaces.webserver import projects as projects_rpc - from simcore_service_api_server.services_rpc import wb_api_server - - # NOTE: mock_missing_plugins patches `setup_rabbitmq` - try: - wb_api_server.WbApiRpcClient.get_from_app_state(app) - except AttributeError: - wb_api_server.setup( - app, RabbitMQRPCClient("fake_rpc_client", settings=mocker.MagicMock()) - ) - - settings: ApplicationSettings = app.state.settings - assert settings.API_SERVER_WEBSERVER +def mocked_webserver_rpc_api(mocker: MockerFixture) -> dict[str, MockType]: + """ + Mocks the webserver's simcore service RPC API for testing purposes. + """ + from servicelib.rabbitmq.rpc_interfaces.webserver import ( + projects as projects_rpc, # keep import here + ) side_effects = WebserverRpcSideEffects() @@ -354,11 +352,13 @@ def mocked_webserver_rpc_api( "mark_project_as_job": mocker.patch.object( projects_rpc, "mark_project_as_job", + autospec=True, side_effect=side_effects.mark_project_as_job, ), "list_projects_marked_as_jobs": mocker.patch.object( projects_rpc, "list_projects_marked_as_jobs", + autospec=True, side_effect=side_effects.list_projects_marked_as_jobs, ), } @@ -458,20 +458,16 @@ def mocked_catalog_rest_api_base( @pytest.fixture -def mocked_catalog_rpc_api( - app: FastAPI, mocker: MockerFixture -) -> Iterable[dict[str, MockType]]: +def mocked_catalog_rpc_api(mocker: MockerFixture) -> dict[str, MockType]: """ - Mocks the RPC catalog service API for testing purposes. + Mocks the catalog's simcore service RPC API for testing purposes. """ + from servicelib.rabbitmq.rpc_interfaces.catalog import ( + services as catalog_rpc, # keep import here + ) - def get_mock_rabbitmq_rpc_client(): - return MagicMock() - - app.dependency_overrides[get_rabbitmq_rpc_client] = get_mock_rabbitmq_rpc_client side_effects = CatalogRpcSideEffects() - - yield { + return { "list_services_paginated": mocker.patch.object( catalog_rpc, "list_services_paginated", @@ -503,7 +499,6 @@ def get_mock_rabbitmq_rpc_client(): side_effect=side_effects.get_service_ports, ), } - app.dependency_overrides.pop(get_rabbitmq_rpc_client) # @@ -633,6 +628,7 @@ def clone_project_task(self, request: httpx.Request, *, project_id: str): return self._set_result_and_get_reponse(project_get) def get_result(self, request: httpx.Request, *, task_id: str): + assert request return httpx.Response( status.HTTP_200_OK, json={"data": self._results[task_id]} ) From 6e613544fd68521a7eb7cdafaa4147c38ef7e843 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 25 Apr 2025 21:49:24 +0200 Subject: [PATCH 27/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20dependency=20injec?= =?UTF-8?q?tion=20in=20SolverService=20and=20add=20mocked=5Fapp=5Fdependen?= =?UTF-8?q?cies=20fixture=20for=20improved=20testability=20and=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 8 ++--- .../api/routes/solvers.py | 8 ++--- .../api/routes/solvers_jobs.py | 4 +-- .../tests/unit/_with_db/test_product.py | 6 ++-- services/api-server/tests/unit/conftest.py | 33 ++++++++++++++++++- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 4d1fa8f36cb..c759ea2200e 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -1,7 +1,4 @@ -from typing import Annotated - from common_library.pagination_tools import iter_pagination_params -from fastapi import Depends from models_library.basic_types import VersionStr from models_library.products import ProductName from models_library.projects_nodes import Node @@ -16,7 +13,6 @@ from models_library.users import UserID from packaging.version import Version -from .api.dependencies.webserver_rpc import get_wb_api_rpc_client from .models.api_resources import compose_resource_name from .models.schemas.jobs import Job, JobInputs from .models.schemas.solvers import Solver, SolverKeyId @@ -35,8 +31,8 @@ class SolverService: def __init__( self, - catalog_service: Annotated[CatalogService, Depends()], - webserver_client: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], + catalog_service: CatalogService, + webserver_client: WbApiRpcClient, ): self._catalog_service = catalog_service self._webserver_client = webserver_client diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 03cc24757f5..beeb6f03707 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -17,7 +17,7 @@ from ...services_http.catalog import CatalogApi from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name -from ..dependencies.services import get_api_client +from ..dependencies.services import get_api_client, get_solver_service from ..dependencies.webserver_http import AuthSession, get_webserver_session from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import ( @@ -153,7 +153,7 @@ async def get_solvers_releases_page( async def get_solver( solver_key: SolverKeyId, user_id: Annotated[int, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): @@ -232,10 +232,10 @@ async def get_solver_release( solver_key: SolverKeyId, version: VersionStr, user_id: Annotated[int, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], -) -> Solver: +): """Gets a specific release of a solver""" try: solver: Solver = await solver_service.get_solver( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 37c97d45e86..7e6621919c6 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -35,7 +35,7 @@ from ...services_rpc.wb_api_server import WbApiRpcClient from ..dependencies.application import get_reverse_url_mapper from ..dependencies.authentication import get_current_user_id, get_product_name -from ..dependencies.services import get_api_client +from ..dependencies.services import get_api_client, get_solver_service from ..dependencies.webserver_http import AuthSession, get_webserver_session from ..dependencies.webserver_rpc import ( get_wb_api_rpc_client, @@ -95,7 +95,7 @@ async def create_solver_job( version: VersionStr, inputs: JobInputs, user_id: Annotated[PositiveInt, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends()], + solver_service: Annotated[SolverService, Depends(get_solver_service)], job_service: Annotated[JobService, Depends()], wb_api_rpc: Annotated[WbApiRpcClient, Depends(get_wb_api_rpc_client)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index 775511713a2..0d73159db27 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -33,10 +33,10 @@ async def test_product_webserver( assert client wallet_to_api_keys_map: dict[int, ApiKeyInDB] = {} - wallet_id: int = faker.pyint(min_value=1) + wid: int = faker.pyint(min_value=1) async for api_key in create_fake_api_keys(2): - wallet_id += faker.pyint(min_value=1) - wallet_to_api_keys_map[wallet_id] = api_key + wid += faker.pyint(min_value=1) + wallet_to_api_keys_map[wid] = api_key def _check_key_product_compatibility(request: httpx.Request, **kwargs): assert ( diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 9a616413a4c..d30085ba29f 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -458,7 +458,38 @@ def mocked_catalog_rest_api_base( @pytest.fixture -def mocked_catalog_rpc_api(mocker: MockerFixture) -> dict[str, MockType]: +def mocked_app_dependencies(app: FastAPI, mocker: MockerFixture) -> Iterator[None]: + """ + Mocks some dependency overrides for the FastAPI app. + """ + from simcore_service_api_server.api.dependencies.rabbitmq import ( + get_rabbitmq_rpc_client, + ) + from simcore_service_api_server.api.dependencies.webserver_rpc import ( + get_wb_api_rpc_client, + ) + + def _get_rabbitmq_rpc_client_override(app: FastAPI): + return mocker.MagicMock() + + async def _get_wb_api_rpc_client_override(app: FastAPI): + return mocker.MagicMock() + + app.dependency_overrides[get_rabbitmq_rpc_client] = ( + _get_rabbitmq_rpc_client_override + ) + app.dependency_overrides[get_wb_api_rpc_client] = _get_wb_api_rpc_client_override + + yield + + app.dependency_overrides.pop(get_wb_api_rpc_client, None) + app.dependency_overrides.pop(get_rabbitmq_rpc_client, None) + + +@pytest.fixture +def mocked_catalog_rpc_api( + mocked_app_dependencies: None, mocker: MockerFixture +) -> dict[str, MockType]: """ Mocks the catalog's simcore service RPC API for testing purposes. """ From af8ebf8aa8a37ea41acef44fb193200aa7e1d941 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 14:36:51 +0200 Subject: [PATCH 28/48] =?UTF-8?q?=E2=9C=A8=20Refactor=20mocked=5Fapp=5Fdep?= =?UTF-8?q?endencies=20fixture=20and=20replace=20mocked=5Fwebserver=5Frpc?= =?UTF-8?q?=5Fapi=20for=20improved=20clarity=20and=20consistency=20in=20te?= =?UTF-8?q?sting=20dependencies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 1 - services/api-server/tests/unit/conftest.py | 100 +++++++++--------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index c759ea2200e..e93e8898e54 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -64,7 +64,6 @@ async def get_latest_release( solver_key: SolverKeyId, product_name: str, ) -> Solver: - # TODO: Mads, this is not necessary. The first item is the latest! service_releases: list[ServiceRelease] = [] for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): releases, page_meta = await self._catalog_service.list_release_history( diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index d30085ba29f..faa5b94ebc1 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -227,6 +227,37 @@ def mocked_s3_server_url() -> Iterator[HttpUrl]: ## MOCKED res/web APIs from simcore services ------------------------------------------ +@pytest.fixture +def mocked_app_dependencies(app: FastAPI, mocker: MockerFixture) -> Iterator[None]: + """ + Mocks some dependency overrides for the FastAPI app. + """ + assert app.state.settings.API_SERVER_RABBITMQ is None + + from simcore_service_api_server.api.dependencies.rabbitmq import ( + get_rabbitmq_rpc_client, + ) + from simcore_service_api_server.api.dependencies.webserver_rpc import ( + get_wb_api_rpc_client, + ) + + def _get_rabbitmq_rpc_client_override(): + return mocker.MagicMock() + + async def _get_wb_api_rpc_client_override(): + return mocker.AsyncMock() + + app.dependency_overrides[get_rabbitmq_rpc_client] = ( + _get_rabbitmq_rpc_client_override + ) + app.dependency_overrides[get_wb_api_rpc_client] = _get_wb_api_rpc_client_override + + yield + + app.dependency_overrides.pop(get_wb_api_rpc_client, None) + app.dependency_overrides.pop(get_rabbitmq_rpc_client, None) + + @pytest.fixture def directorv2_service_openapi_specs( osparc_simcore_services_dir: Path, @@ -337,33 +368,6 @@ def mocked_webserver_rest_api_base( yield respx_mock -@pytest.fixture -def mocked_webserver_rpc_api(mocker: MockerFixture) -> dict[str, MockType]: - """ - Mocks the webserver's simcore service RPC API for testing purposes. - """ - from servicelib.rabbitmq.rpc_interfaces.webserver import ( - projects as projects_rpc, # keep import here - ) - - side_effects = WebserverRpcSideEffects() - - return { - "mark_project_as_job": mocker.patch.object( - projects_rpc, - "mark_project_as_job", - autospec=True, - side_effect=side_effects.mark_project_as_job, - ), - "list_projects_marked_as_jobs": mocker.patch.object( - projects_rpc, - "list_projects_marked_as_jobs", - autospec=True, - side_effect=side_effects.list_projects_marked_as_jobs, - ), - } - - @pytest.fixture def mocked_storage_rest_api_base( app: FastAPI, @@ -458,32 +462,32 @@ def mocked_catalog_rest_api_base( @pytest.fixture -def mocked_app_dependencies(app: FastAPI, mocker: MockerFixture) -> Iterator[None]: +def mocked_webserver_rpc_api( + mocked_app_dependencies: None, mocker: MockerFixture +) -> dict[str, MockType]: """ - Mocks some dependency overrides for the FastAPI app. + Mocks the webserver's simcore service RPC API for testing purposes. """ - from simcore_service_api_server.api.dependencies.rabbitmq import ( - get_rabbitmq_rpc_client, - ) - from simcore_service_api_server.api.dependencies.webserver_rpc import ( - get_wb_api_rpc_client, - ) - - def _get_rabbitmq_rpc_client_override(app: FastAPI): - return mocker.MagicMock() - - async def _get_wb_api_rpc_client_override(app: FastAPI): - return mocker.MagicMock() - - app.dependency_overrides[get_rabbitmq_rpc_client] = ( - _get_rabbitmq_rpc_client_override + from servicelib.rabbitmq.rpc_interfaces.webserver import ( + projects as projects_rpc, # keep import here ) - app.dependency_overrides[get_wb_api_rpc_client] = _get_wb_api_rpc_client_override - yield + side_effects = WebserverRpcSideEffects() - app.dependency_overrides.pop(get_wb_api_rpc_client, None) - app.dependency_overrides.pop(get_rabbitmq_rpc_client, None) + return { + "mark_project_as_job": mocker.patch.object( + projects_rpc, + "mark_project_as_job", + autospec=True, + side_effect=side_effects.mark_project_as_job, + ), + "list_projects_marked_as_jobs": mocker.patch.object( + projects_rpc, + "list_projects_marked_as_jobs", + autospec=True, + side_effect=side_effects.list_projects_marked_as_jobs, + ), + } @pytest.fixture From f2555f3a9ac4a940d9b96be0cb9453cfb62e93be Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 14:40:32 +0200 Subject: [PATCH 29/48] =?UTF-8?q?=E2=9C=A8=20Add=20trim=5Fstring=5Fbefore?= =?UTF-8?q?=20pre-validator=20and=20corresponding=20tests=20for=20improved?= =?UTF-8?q?=20string=20handling=20in=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_utils_common_validators.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py index db9df708b0f..46175fd5f2d 100644 --- a/packages/models-library/tests/test_utils_common_validators.py +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import Annotated import pytest from models_library.utils.common_validators import ( @@ -6,6 +7,7 @@ empty_str_to_none_pre_validator, none_to_empty_str_pre_validator, null_or_none_str_to_none_validator, + trim_string_before, ) from pydantic import BaseModel, ValidationError, field_validator @@ -89,3 +91,33 @@ class Model(BaseModel): model = Model.model_validate({"message": ""}) assert model == Model.model_validate({"message": ""}) + + +def test_trim_string_before(): + max_length = 10 + + class ModelWithTrim(BaseModel): + text: Annotated[str, trim_string_before(max_length=max_length)] + + # Test with string shorter than max_length + short_text = "Short" + model = ModelWithTrim(text=short_text) + assert model.text == short_text + + # Test with string equal to max_length + exact_text = "1234567890" # 10 characters + model = ModelWithTrim(text=exact_text) + assert model.text == exact_text + + # Test with string longer than max_length + long_text = "This is a very long text that should be trimmed" + model = ModelWithTrim(text=long_text) + assert model.text == long_text[:max_length] + assert len(model.text) == max_length + + # Test with non-string value (should be left unchanged) + class ModelWithTrimOptional(BaseModel): + text: Annotated[str | None, trim_string_before(max_length=max_length)] + + model = ModelWithTrimOptional(text=None) + assert model.text is None From 6cf1f8a99b2f6f822e5faf3bf003a168abbc485b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 16:27:31 +0200 Subject: [PATCH 30/48] cleanup tests --- services/api-server/tests/unit/test_api_solver_jobs.py | 10 +++++----- services/api-server/tests/unit/test_api_solvers.py | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/services/api-server/tests/unit/test_api_solver_jobs.py b/services/api-server/tests/unit/test_api_solver_jobs.py index a618cd6b856..518d183603d 100644 --- a/services/api-server/tests/unit/test_api_solver_jobs.py +++ b/services/api-server/tests/unit/test_api_solver_jobs.py @@ -40,7 +40,7 @@ def _start_job_side_effect( return capture.response_body -def get_inspect_job_side_effect(job_id: str) -> SideEffectCallback: +def _get_inspect_job_side_effect(job_id: str) -> SideEffectCallback: def _inspect_job_side_effect( request: httpx.Request, path_params: dict[str, Any], @@ -250,7 +250,7 @@ def _put_pricing_plan_and_unit_side_effect( _start_job_side_effect, ] if expected_status_code == status.HTTP_202_ACCEPTED: - callbacks.append(get_inspect_job_side_effect(job_id=_job_id)) + callbacks.append(_get_inspect_job_side_effect(job_id=_job_id)) _put_pricing_plan_and_unit_side_effect.was_called = False create_respx_mock_from_capture( @@ -296,7 +296,7 @@ async def test_get_solver_job_pricing_unit_no_payment( capture_path=project_tests_dir / "mocks" / "start_job_no_payment.json", side_effects_callbacks=[ _start_job_side_effect, - get_inspect_job_side_effect(job_id=_job_id), + _get_inspect_job_side_effect(job_id=_job_id), ], ) @@ -329,7 +329,7 @@ async def test_start_solver_job_conflict( capture_path=project_tests_dir / "mocks" / "start_solver_job.json", side_effects_callbacks=[ _start_job_side_effect, - get_inspect_job_side_effect(job_id=_job_id), + _get_inspect_job_side_effect(job_id=_job_id), ], ) @@ -370,7 +370,7 @@ def _stop_job_side_effect( capture_path=project_tests_dir / "mocks" / "stop_job.json", side_effects_callbacks=[ _stop_job_side_effect, - get_inspect_job_side_effect(job_id=_job_id), + _get_inspect_job_side_effect(job_id=_job_id), ], ) diff --git a/services/api-server/tests/unit/test_api_solvers.py b/services/api-server/tests/unit/test_api_solvers.py index 0f76b520e01..9d0eac22ac0 100644 --- a/services/api-server/tests/unit/test_api_solvers.py +++ b/services/api-server/tests/unit/test_api_solvers.py @@ -7,9 +7,11 @@ import httpx import pytest +import respx from fastapi import status from httpx import AsyncClient from models_library.api_schemas_api_server.pricing_plans import ServicePricingPlanGet +from pytest_mock import MockType from pytest_simcore.helpers.httpx_calls_capture_models import CreateRespxMockCallback from simcore_service_api_server._meta import API_VTAG @@ -26,7 +28,7 @@ ) async def test_get_solver_pricing_plan( client: AsyncClient, - mocked_webserver_rest_api_base, + mocked_webserver_rest_api_base: respx.MockRouter, create_respx_mock_from_capture: CreateRespxMockCallback, auth: httpx.BasicAuth, project_tests_dir: Path, @@ -60,7 +62,7 @@ async def test_get_solver_pricing_plan( ) async def test_get_latest_solver_release( client: AsyncClient, - mocked_catalog_rpc_api, + mocked_catalog_rpc_api: dict[str, MockType], auth: httpx.BasicAuth, solver_key: str, expected_status_code: int, From 1c62cdbda154eb98245f646a7db1d62522e528a4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 16:33:23 +0200 Subject: [PATCH 31/48] drafting tests --- .../test_api_routers_solvers_jobs_read.py | 57 +++++++++++++++++-- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index 36f02845525..7209801a999 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -12,19 +12,22 @@ from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG +from simcore_service_api_server.api.dependencies import webserver_rpc from simcore_service_api_server.models.pagination import Page from simcore_service_api_server.models.schemas.jobs import Job from starlette import status class MockBackendRouters(NamedTuple): - webserver: MockRouter - catalog: dict[str, MockType] + webserver_rest: MockRouter + webserver_rpc: dict[str, MockType] + catalog_rpc: dict[str, MockType] @pytest.fixture def mocked_backend( mocked_webserver_rest_api_base: MockRouter, + mocked_webserver_rpc_api: dict[str, MockType], mocked_catalog_rpc_api: dict[str, MockType], project_tests_dir: Path, ) -> MockBackendRouters: @@ -46,8 +49,9 @@ def mocked_backend( ) return MockBackendRouters( - webserver=mocked_webserver_rest_api_base, - catalog=mocked_catalog_rpc_api, + webserver_rest=mocked_webserver_rest_api_base, + webserver_rpc=mocked_webserver_rpc_api, + catalog_rpc=mocked_catalog_rpc_api, ) @@ -81,5 +85,46 @@ async def test_list_solver_jobs( assert jobs_page.items == jobs # check calls to the deep-backend services - assert mocked_backend.webserver["list_projects"].called - assert mocked_backend.catalog["get_service"].called + assert mocked_backend.webserver_rest["list_projects"].called + assert mocked_backend.catalog_rpc["get_service"].called + + +async def test_list_all_solvers_jobs( + auth: httpx.BasicAuth, + client: httpx.AsyncClient, + mocked_backend: MockBackendRouters, +): + """Tests the endpoint that lists all jobs across all solvers.""" + + # Call the endpoint with pagination parameters + resp = await client.get( + f"/{API_VTAG}/solvers/-/releases/-/jobs", + auth=auth, + params={"limit": 10, "offset": 0}, + ) + + # Verify the response + assert resp.status_code == status.HTTP_200_OK + + # Parse and validate the response + jobs_page = TypeAdapter(Page[Job]).validate_python(resp.json()) + + # Basic assertions on the response structure + assert isinstance(jobs_page.items, list) + assert hasattr(jobs_page, "meta") + assert hasattr(jobs_page.meta, "total") + + # Each job should have the expected structure + for job in jobs_page.items: + assert isinstance(job.id, str) + assert isinstance(job.name, str) + assert hasattr(job, "url") + + # Verify interactions with backend services + # These will need to be adjusted based on how the endpoint is implemented + # For now, we can assume it might use the solver_service's list_jobs method + + # Additional tests could include: + # - Testing pagination by retrieving different pages + # - Testing with different limit/offset parameters + # - Checking that jobs from different solvers are included From 1f88d50e49bc096fc7c69b2c0ea9029c30160c10 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:07:15 +0200 Subject: [PATCH 32/48] adds first tests --- .../models_library/rpc/webserver/projects.py | 24 ++++++++++++++--- .../helpers/webserver_rpc_server.py | 9 ++++++- .../test_api_routers_solvers_jobs_read.py | 26 ++++++++----------- services/api-server/tests/unit/conftest.py | 3 ++- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/webserver/projects.py b/packages/models-library/src/models_library/rpc/webserver/projects.py index 21ca6a8310d..195358a7213 100644 --- a/packages/models-library/src/models_library/rpc/webserver/projects.py +++ b/packages/models-library/src/models_library/rpc/webserver/projects.py @@ -43,12 +43,30 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "examples": [ { "uuid": "12345678-1234-5678-1234-123456789012", - "name": "My project", - "description": "My project description", + "name": "A solver job", + "description": "A description of a solver job with a single node", "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, "created_at": "2023-01-01T00:00:00Z", "modified_at": "2023-01-01T00:00:00Z", - "job_parent_resource_name": "solvers/slv_123/release/1.2.3", + "job_parent_resource_name": "solvers/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/releases/2.0.2", + }, + { + "uuid": "00000000-1234-5678-1234-123456789012", + "name": "A study job", + "description": "A description of a study job with many node", + "workbench": {f"{uuid4()}": n for n in nodes_examples}, + "created_at": "2023-02-01T00:00:00Z", + "modified_at": "2023-02-01T00:00:00Z", + "job_parent_resource_name": "studies/96642f2a-a72c-11ef-8776-02420a00087d", + }, + { + "uuid": "00000000-0000-5678-1234-123456789012", + "name": "A program job", + "description": "A program of a solver job with a single node", + "workbench": {f"{uuid4()}": n for n in nodes_examples[2:3]}, + "created_at": "2023-03-01T00:00:00Z", + "modified_at": "2023-03-01T00:00:00Z", + "job_parent_resource_name": "program/simcore%2Fservices%2Fdynamic%2Fjupyter/releases/5.0.2", }, ] } diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py index 13617f25ab1..ca645218579 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_rpc_server.py @@ -68,7 +68,14 @@ async def list_projects_marked_as_jobs( assert not job_parent_resource_name_prefix.endswith("%") assert not job_parent_resource_name_prefix.startswith("%") - items = ProjectJobRpcGet.model_json_schema()["examples"] + items = [ + item + for item in ProjectJobRpcGet.model_json_schema()["examples"] + if job_parent_resource_name_prefix is None + or item.get("job_parent_resource_name").startswith( + job_parent_resource_name_prefix + ) + ] return PageRpcProjectJobRpcGet.create( items[offset : offset + limit], diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py index 7209801a999..b7dca6511a6 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers_jobs_read.py @@ -12,7 +12,6 @@ from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel from respx import MockRouter from simcore_service_api_server._meta import API_VTAG -from simcore_service_api_server.api.dependencies import webserver_rpc from simcore_service_api_server.models.pagination import Page from simcore_service_api_server.models.schemas.jobs import Job from starlette import status @@ -111,20 +110,17 @@ async def test_list_all_solvers_jobs( # Basic assertions on the response structure assert isinstance(jobs_page.items, list) - assert hasattr(jobs_page, "meta") - assert hasattr(jobs_page.meta, "total") + assert jobs_page.total > 0 + assert jobs_page.limit == 10 + assert jobs_page.offset == 0 + assert jobs_page.total <= len(jobs_page.items) # Each job should have the expected structure for job in jobs_page.items: - assert isinstance(job.id, str) - assert isinstance(job.name, str) - assert hasattr(job, "url") - - # Verify interactions with backend services - # These will need to be adjusted based on how the endpoint is implemented - # For now, we can assume it might use the solver_service's list_jobs method - - # Additional tests could include: - # - Testing pagination by retrieving different pages - # - Testing with different limit/offset parameters - # - Checking that jobs from different solvers are included + assert job.id + assert job.name + assert job.url is not None + assert job.runner_url is not None + assert job.outputs_url is not None + + assert mocked_backend.webserver_rpc["list_projects_marked_as_jobs"].called diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index faa5b94ebc1..3389a9b4d2f 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -50,6 +50,7 @@ from simcore_service_api_server.core.settings import ApplicationSettings from simcore_service_api_server.repository.api_keys import UserAndProductTuple from simcore_service_api_server.services_http.solver_job_outputs import ResultsTypes +from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient @pytest.fixture @@ -245,7 +246,7 @@ def _get_rabbitmq_rpc_client_override(): return mocker.MagicMock() async def _get_wb_api_rpc_client_override(): - return mocker.AsyncMock() + return WbApiRpcClient(_client=mocker.MagicMock()) app.dependency_overrides[get_rabbitmq_rpc_client] = ( _get_rabbitmq_rpc_client_override From f2e646a7e99be1799eaa922e93f51df2819e2e1e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:48:28 +0200 Subject: [PATCH 33/48] drafting doc --- .../api-server/docs/api-server.drawio.svg | 745 ++++++++++++++++++ 1 file changed, 745 insertions(+) create mode 100644 services/api-server/docs/api-server.drawio.svg diff --git a/services/api-server/docs/api-server.drawio.svg b/services/api-server/docs/api-server.drawio.svg new file mode 100644 index 00000000000..440d1dcaaa7 --- /dev/null +++ b/services/api-server/docs/api-server.drawio.svg @@ -0,0 +1,745 @@ + + + + + + + + + + + + +
+
+
+ CONTROLLER +
+
+
+ + + CONTROLLER + + + + + + + + + + + +
+
+
+ SERVICE +
+
+
+ + + SERVICE + + + + + + + + + + + +
+
+
+ REPOSITORY +
+
+
+ + + REPOSITORY + + + + + + + + + + + +
+
+
+ CLIENTS +
+
+
+ + + CLIENTS + + + + + + + + + + + + +
+
+
+ CONTROLLER +
+
+
+ + + CONTROLLER + + + + + + + + + + + +
+
+
+ SERVICE +
+
+
+ + + SERVICE + + + + + + + + + + + +
+
+
+ REPOSITORY +
+
+
+ + + REPOSITORY + + + + + + + + + + + +
+
+
+ CLIENTS +
+
+
+ + + CLIENTS + + + + + + + + + + + + + + + +
+
+
+ rest +
+
+
+ + + rest + + + + + + + + + + + + + + + +
+
+
+ rpc +
+
+
+ + + rpc + + + + + + + + + + + +
+
+
+ projects +
+
+
+ + + projects + + + + + + + + + + + + +
+
+
+ CONTROLLER +
+
+
+ + + CONTROLLER + + + + + + + + + + + +
+
+
+ SERVICE +
+
+
+ + + SERVICE + + + + + + + + + + + +
+
+
+ REPOSITORY +
+
+
+ + + REPOSITORY + + + + + + + + + + + +
+
+
+ CLIENTS +
+
+
+ + + CLIENTS + + + + + + + + + + + +
+
+
+ rest +
+
+
+ + + rest + + + + + + + + + + + +
+
+
+ rpc +
+
+
+ + + rpc + + + + + + + + + + + + +
+
+
+ postgres +
+
+
+ + + postgres + + + + + + + + + + + + + + + + + + + +
+
+
+ SolverService +
+
+
+ + + SolverService + + + + + + + + + + + +
+
+
+ CatalogService +
+
+
+ + + CatalogService + + + + + + + + + + + + +
+
+
+ rabbitmq-rpc +
+
+
+ + + rabbitmq-rpc + + + + + + + + + + + + +
+
+
+ httpx +
+
+
+ + + httpx + + + + + + + + + + + +
+
+
+ AuthSession +
+
+
+ + + AuthSession + + + + + + + + + + + + +
+
+
+ rabbitmq-rpc +
+
+
+ + + rabbitmq-rpc + + + + + + + + + + + +
+
+
+ WbApiRpcClient +
+
+
+ + + WbApiRpcClient + + + + + + + + + + + + +
+
+
+ asyncpg +
+
+
+ + + asyncpg + + + + + + + + + + + +
+
+
+ AsyncEngine +
+
+
+ + + AsyncEngine + + + + + + + + + + + +
+
+
+ simcore_service_catalog +
+
+
+ + + simcore_ser... + + + + + + + + + + + +
+
+
+ simcore_service_webserver +
+
+
+ + + simcore_ser... + + + + + + + + + + + +
+
+
+ simcore_service_api_server +
+
+
+ + + simcore_ser... + + + + + + + + + + + + + + + +
+
+
+ /solvers +
+
+
+ + + /solvers + + + + + + + + + + + + + + + +
+
+
+ ProgramsService +
+
+
+ + + ProgramsService + + + + + + + + + + + + + + + +
+
+
+ /programs +
+
+
+ + + /programs + + + + + + + + + + Text is not SVG - cannot display + + + + From cfb0fc40db863c378d2a0350f613917d15e33298 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:52:15 +0200 Subject: [PATCH 34/48] updates OAS --- services/api-server/openapi.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 9c0d63ed8e9..d4cf4ba7406 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -7692,7 +7692,7 @@ "anyOf": [ { "type": "string", - "maxLength": 500 + "maxLength": 1000 }, { "type": "null" @@ -7823,7 +7823,7 @@ "anyOf": [ { "type": "string", - "maxLength": 500 + "maxLength": 1000 }, { "type": "null" From dddec2d071fd804bce1156d5e24f47a2206ff318 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 17:56:40 +0200 Subject: [PATCH 35/48] updates tests --- .../tests/test_utils_common_validators.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py index 46175fd5f2d..76353730e8b 100644 --- a/packages/models-library/tests/test_utils_common_validators.py +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -9,7 +9,7 @@ null_or_none_str_to_none_validator, trim_string_before, ) -from pydantic import BaseModel, ValidationError, field_validator +from pydantic import BaseModel, StringConstraints, ValidationError, field_validator def test_enums_pre_validator(): @@ -121,3 +121,31 @@ class ModelWithTrimOptional(BaseModel): model = ModelWithTrimOptional(text=None) assert model.text is None + + +def test_trim_string_before_with_string_constraints(): + max_length = 10 + + class ModelWithTrimAndConstraints(BaseModel): + text: Annotated[ + str, + trim_string_before(max_length=max_length), + StringConstraints(max_length=max_length), + ] + + # Check that the OpenAPI schema contains the string constraint + schema = ModelWithTrimAndConstraints.model_json_schema() + assert schema["properties"]["text"]["maxLength"] == max_length + + # Test with string longer than max_length + # This should pass because trim_string_before runs first and trims the input + # before StringConstraints validation happens + long_text = "This is a very long text that should be trimmed" + model = ModelWithTrimAndConstraints(text=long_text) + assert model.text == long_text[:max_length] + assert len(model.text) == max_length + + # Test with string exactly at max_length + exact_text = "1234567890" # 10 characters + model = ModelWithTrimAndConstraints(text=exact_text) + assert model.text == exact_text From 0886a565b62f31cfc68443345491f94079c3dc79 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 28 Apr 2025 22:11:09 +0200 Subject: [PATCH 36/48] tests --- .../tests/test_utils_common_validators.py | 12 +++++++++--- .../models/schemas/_base.py | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/models-library/tests/test_utils_common_validators.py b/packages/models-library/tests/test_utils_common_validators.py index 76353730e8b..5212f5d5bab 100644 --- a/packages/models-library/tests/test_utils_common_validators.py +++ b/packages/models-library/tests/test_utils_common_validators.py @@ -128,20 +128,26 @@ def test_trim_string_before_with_string_constraints(): class ModelWithTrimAndConstraints(BaseModel): text: Annotated[ - str, + str | None, + StringConstraints( + max_length=max_length + ), # NOTE: order does not matter for validation but has an effect in the openapi schema trim_string_before(max_length=max_length), - StringConstraints(max_length=max_length), ] # Check that the OpenAPI schema contains the string constraint schema = ModelWithTrimAndConstraints.model_json_schema() - assert schema["properties"]["text"]["maxLength"] == max_length + assert schema["properties"]["text"] == { + "anyOf": [{"maxLength": max_length, "type": "string"}, {"type": "null"}], + "title": "Text", + } # Test with string longer than max_length # This should pass because trim_string_before runs first and trims the input # before StringConstraints validation happens long_text = "This is a very long text that should be trimmed" model = ModelWithTrimAndConstraints(text=long_text) + assert model.text is not None assert model.text == long_text[:max_length] assert len(model.text) == max_length diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py index 187de8ce20f..fb2fd0e6a25 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py @@ -45,8 +45,10 @@ class BaseService(BaseModel): ] description: Annotated[ str | None, + StringConstraints( # NOTE: keep StringConstraints before to keep the openapi schema + max_length=1000 + ), trim_string_before(max_length=1000), - StringConstraints(max_length=1000), Field(default=None, description="Description of the resource"), ] From accb75c3d71aacfda00b6ec5a2c462f62aecc32e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:42:51 +0200 Subject: [PATCH 37/48] @bisgaard-itis review: solver arguments --- .../_service_solvers.py | 20 +++++++++---------- .../api/routes/solvers.py | 4 ++-- .../api/routes/solvers_jobs.py | 4 ++-- .../api/routes/solvers_jobs_getters.py | 12 +++++------ .../tests/unit/test_service_solvers.py | 4 ++-- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index e93e8898e54..759e99c85c4 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -40,15 +40,15 @@ def __init__( async def get_solver( self, *, - user_id: UserID, - name: SolverKeyId, - version: VersionStr, product_name: ProductName, + user_id: UserID, + solver_key: SolverKeyId, + solver_version: VersionStr, ) -> Solver: service = await self._catalog_service.get( user_id=user_id, - name=name, - version=version, + name=solver_key, + version=solver_version, product_name=product_name, ) assert ( # nosec @@ -60,9 +60,9 @@ async def get_solver( async def get_latest_release( self, *, + product_name: str, user_id: int, solver_key: SolverKeyId, - product_name: str, ) -> Solver: service_releases: list[ServiceRelease] = [] for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): @@ -89,10 +89,10 @@ async def get_latest_release( async def list_jobs( self, *, - user_id: UserID, product_name: ProductName, + user_id: UserID, # filters - solver_id: SolverKeyId | None = None, + solver_key: SolverKeyId | None = None, solver_version: VersionStr | None = None, # pagination offset: PageOffsetInt = 0, @@ -104,8 +104,8 @@ async def list_jobs( collection_or_resource_ids = [ "solvers", # solver_id, "releases", solver_version, "jobs", ] - if solver_id: - collection_or_resource_ids.append(solver_id) + if solver_key: + collection_or_resource_ids.append(solver_key) if solver_version: collection_or_resource_ids.append("releases") collection_or_resource_ids.append(solver_version) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index beeb6f03707..365cd0b8c1b 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -240,8 +240,8 @@ async def get_solver_release( try: solver: Solver = await solver_service.get_solver( user_id=user_id, - name=solver_key, - version=version, + solver_key=solver_key, + solver_version=version, product_name=product_name, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py index 7e6621919c6..aaa1b8aad7c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py @@ -112,8 +112,8 @@ async def create_solver_job( # ensures user has access to solver solver = await solver_service.get_solver( user_id=user_id, - name=solver_key, - version=version, + solver_key=solver_key, + solver_version=version, product_name=product_name, ) job, project = await job_service.create_job( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index df19d68512d..f130f02ad0c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -185,8 +185,8 @@ async def list_jobs( solver = await solver_service.get_solver( user_id=user_id, - name=solver_key, - version=version, + solver_key=solver_key, + solver_version=version, product_name=product_name, ) _logger.debug("Listing Jobs in Solver '%s'", solver.name) @@ -232,8 +232,8 @@ async def get_jobs_page( solver = await solver_service.get_solver( user_id=user_id, - name=solver_key, - version=version, + solver_key=solver_key, + solver_version=version, product_name=product_name, ) _logger.debug("Listing Jobs in Solver '%s'", solver.name) @@ -276,8 +276,8 @@ async def get_job( solver = await solver_service.get_solver( user_id=user_id, - name=solver_key, - version=version, + solver_key=solver_key, + solver_version=version, product_name=product_name, ) project: ProjectGet = await webserver_api.get_project(project_id=job_id) diff --git a/services/api-server/tests/unit/test_service_solvers.py b/services/api-server/tests/unit/test_service_solvers.py index 35655fdfd3d..d0fe16405a5 100644 --- a/services/api-server/tests/unit/test_service_solvers.py +++ b/services/api-server/tests/unit/test_service_solvers.py @@ -33,8 +33,8 @@ async def test_get_solver( ): solver = await solver_service.get_solver( user_id=user_id, - name="simcore/services/comp/solver-1", - version="1.0.0", + solver_key="simcore/services/comp/solver-1", + solver_version="1.0.0", product_name=product_name, ) From 79c08cb07e668988a126f0b4b1a72dd3e68741ef Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:47:17 +0200 Subject: [PATCH 38/48] @bisgaard-itis review: dev feature --- .../api/routes/solvers_jobs_getters.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index f130f02ad0c..38015715c37 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -58,7 +58,6 @@ from ..dependencies.rabbitmq import get_log_check_timeout, get_log_distributor from ..dependencies.services import get_api_client, get_solver_service from ..dependencies.webserver_http import AuthSession, get_webserver_session -from ._common import API_SERVER_DEV_FEATURES_ENABLED from ._constants import ( FMSG_CHANGELOG_NEW_IN_VERSION, FMSG_CHANGELOG_REMOVED_IN_VERSION_FORMAT, @@ -126,8 +125,13 @@ @router.get( "/-/releases/-/jobs", response_model=Page[Job], - description="List of all jobs created for any released solver (paginated)", - include_in_schema=API_SERVER_DEV_FEATURES_ENABLED, + description=create_route_description( + base="List of all jobs created for any released solver (paginated)", + changelog=[ + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.8"), + ], + ), + include_in_schema=False, # TO BE RELEASED in 0.8 ) async def list_all_solvers_jobs( user_id: Annotated[PositiveInt, Depends(get_current_user_id)], From b31b233223a108ae79582b2fc835fb220c0939ce Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 09:54:48 +0200 Subject: [PATCH 39/48] @bisgaard-itis review: tuples --- .../models/api_resources.py | 17 +++++++++++------ .../tests/unit/test_models_api_resources.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/models/api_resources.py b/services/api-server/src/simcore_service_api_server/models/api_resources.py index 2ae99972e51..939012bbf57 100644 --- a/services/api-server/src/simcore_service_api_server/models/api_resources.py +++ b/services/api-server/src/simcore_service_api_server/models/api_resources.py @@ -53,26 +53,31 @@ def compose_resource_name(*collection_or_resource_ids) -> RelativeResourceName: return TypeAdapter(RelativeResourceName).validate_python("/".join(quoted_parts)) -def split_resource_name(resource_name: RelativeResourceName) -> list[str]: +def split_resource_name(resource_name: RelativeResourceName) -> tuple[str, ...]: + """ + Example: + resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" + returns ("solvers", "simcore/services/comp/isolve", "releases", "1.3.4", "jobs", "f622946d-fd29-35b9-a193-abdd1095167c", "outputs", "output 22") + """ quoted_parts = resource_name.split("/") - return [f"{urllib.parse.unquote_plus(p)}" for p in quoted_parts] + return tuple(f"{urllib.parse.unquote_plus(p)}" for p in quoted_parts) -def parse_collections_ids(resource_name: RelativeResourceName) -> list[str]: +def parse_collections_ids(resource_name: RelativeResourceName) -> tuple[str, ...]: """ Example: resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" - returns ["solvers", "releases", "jobs", "outputs"] + returns ("solvers", "releases", "jobs", "outputs") """ parts = split_resource_name(resource_name) return parts[::2] -def parse_resources_ids(resource_name: RelativeResourceName) -> list[str]: +def parse_resources_ids(resource_name: RelativeResourceName) -> tuple[str, ...]: """ Example: resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" - returns ["simcore/services/comp/isolve", "1.3.4", "f622946d-fd29-35b9-a193-abdd1095167c", "output 22"] + returns ("simcore/services/comp/isolve", "1.3.4", "f622946d-fd29-35b9-a193-abdd1095167c", "output 22") """ parts = split_resource_name(resource_name) return parts[1::2] diff --git a/services/api-server/tests/unit/test_models_api_resources.py b/services/api-server/tests/unit/test_models_api_resources.py index 39137bcb8d0..d204b32215d 100644 --- a/services/api-server/tests/unit/test_models_api_resources.py +++ b/services/api-server/tests/unit/test_models_api_resources.py @@ -17,7 +17,7 @@ def test_parse_resource_id(): resource_name = "solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c/outputs/output+22" - parts = [ + parts = ( "solvers", "simcore/services/comp/isolve", "releases", @@ -26,7 +26,7 @@ def test_parse_resource_id(): "f622946d-fd29-35b9-a193-abdd1095167c", "outputs", "output 22", - ] + ) # cannot use this because cannot convert into URL? except {:path} in starlette ??? assert str(Path(*parts)) == urllib.parse.unquote_plus(resource_name) @@ -44,10 +44,10 @@ def test_parse_resource_id(): collection_to_resource_id_map = split_resource_name_as_dict(resource_name) # Collection-ID -> Resource-ID - assert list(collection_to_resource_id_map.keys()) == parse_collections_ids( + assert tuple(collection_to_resource_id_map.keys()) == parse_collections_ids( resource_name ) - assert list(collection_to_resource_id_map.values()) == parse_resources_ids( + assert tuple(collection_to_resource_id_map.values()) == parse_resources_ids( resource_name ) From 3b9300d3ff4f38696ce3aa404ca4590f147d34b0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:02:31 +0200 Subject: [PATCH 40/48] @sanderegg review: back to 2024 :-) --- services/api-server/tests/unit/_with_db/test_product.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-server/tests/unit/_with_db/test_product.py b/services/api-server/tests/unit/_with_db/test_product.py index 0d73159db27..1c672da9133 100644 --- a/services/api-server/tests/unit/_with_db/test_product.py +++ b/services/api-server/tests/unit/_with_db/test_product.py @@ -33,10 +33,10 @@ async def test_product_webserver( assert client wallet_to_api_keys_map: dict[int, ApiKeyInDB] = {} - wid: int = faker.pyint(min_value=1) + _wallet_id = faker.pyint(min_value=1) async for api_key in create_fake_api_keys(2): - wid += faker.pyint(min_value=1) - wallet_to_api_keys_map[wid] = api_key + _wallet_id += faker.pyint(min_value=1) + wallet_to_api_keys_map[_wallet_id] = api_key def _check_key_product_compatibility(request: httpx.Request, **kwargs): assert ( From 233c53eca8d0a3e27573f74813a0a371da51f880 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 10:03:57 +0200 Subject: [PATCH 41/48] @sanderegg review: doc --- .../simcore_service_webserver/projects/_jobs_repository.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index 04ac1a34dec..d2f54aa5726 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -67,7 +67,12 @@ async def list_projects_marked_as_jobs( """ Lists projects marked as jobs for a specific user and product - Example: job_parent_resource_name = "/solvers/solver1" + + `job_parent_resource_name_prefix` is a prefix to filter the `job_parent_resource_name`. The latter is a + path-like string that contains a hierarchy of resources. An example of `job_parent_resource_name` is: + `/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c` + SEE services/api-server/src/simcore_service_api_server/models/api_resources.py + """ # Step 1: Get group IDs associated with the user user_groups_query = ( From ab800922600d66891ba214a7b238ec86182834ce Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:17:20 +0200 Subject: [PATCH 42/48] @matusdrobuliak66 review: pricate workspace --- .../projects/_groups_service.py | 18 +++++++------ .../projects/_jobs_repository.py | 25 ++++++++++++++----- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_service.py b/services/web/server/src/simcore_service_webserver/projects/_groups_service.py index 40416db7975..4ad89126a49 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_service.py @@ -160,14 +160,16 @@ async def delete_project_group( project_db: ProjectDBAPI = app[APP_PROJECT_DBAPI] project = await project_db.get_project_db(project_id) project_owner_user: dict = await users_service.get_user(app, project.prj_owner) - if project_owner_user["primary_gid"] == group_id: - if user["primary_gid"] != project_owner_user["primary_gid"]: - # Only the owner of the project can delete the owner group - raise ProjectInvalidRightsError( - user_id=user_id, - project_uuid=project_id, - reason=f"User does not have access to modify owner project group in project {project_id}", - ) + if ( + project_owner_user["primary_gid"] == group_id + and user["primary_gid"] != project_owner_user["primary_gid"] + ): + # Only the owner of the project can delete the owner group + raise ProjectInvalidRightsError( + user_id=user_id, + project_uuid=project_id, + reason=f"User does not have access to modify owner project group in project {project_id}", + ) await _groups_repository.delete_project_group( app=app, project_id=project_id, group_id=group_id diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index d2f54aa5726..a165623afc4 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -64,16 +64,26 @@ async def list_projects_marked_as_jobs( limit: int = 10, job_parent_resource_name_prefix: str | None = None, ) -> tuple[int, list[ProjectJobDBGet]]: - """ - Lists projects marked as jobs for a specific user and product + """Lists projects marked as jobs for a specific user and product + + Arguments: + product_name -- product identifier of the caller's context + user_id -- user identifier of the caller - `job_parent_resource_name_prefix` is a prefix to filter the `job_parent_resource_name`. The latter is a - path-like string that contains a hierarchy of resources. An example of `job_parent_resource_name` is: - `/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c` - SEE services/api-server/src/simcore_service_api_server/models/api_resources.py + Keyword Arguments: + connection -- (default: {None}) + offset -- pagination offset (default: {0}) + limit -- pagittion limit (default: {10}) + job_parent_resource_name_prefix -- is a prefix to filter the `job_parent_resource_name`. The latter is a + path-like string that contains a hierarchy of resources. An example of `job_parent_resource_name` is: + `/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c` + SEE services/api-server/src/simcore_service_api_server/models/api_resources.py (default: {None}) + Returns: + total_count, list of projects marked as jobs """ + # Step 1: Get group IDs associated with the user user_groups_query = ( sa.select(user_to_groups.c.gid) @@ -98,6 +108,9 @@ async def list_projects_marked_as_jobs( projects_to_products.c.product_name == product_name, project_to_groups.c.gid.in_(sa.select(user_groups_query.c.gid)), project_to_groups.c.read.is_(True), + projects.c.workspace_id.is_( + None + ), # ONLY projects in private workspaces ) ) From 47fae42c8f31b4bb7c010b0bb2d29b66e3ea0dc9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:17:28 +0200 Subject: [PATCH 43/48] @matusdrobuliak66 review: pricate workspace --- .../projects/_jobs_repository.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py index a165623afc4..b6fd3242c01 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py @@ -68,13 +68,10 @@ async def list_projects_marked_as_jobs( Arguments: - product_name -- product identifier of the caller's context - user_id -- user identifier of the caller + product_name -- caller's context product identifier + user_id -- caller's user identifier Keyword Arguments: - connection -- (default: {None}) - offset -- pagination offset (default: {0}) - limit -- pagittion limit (default: {10}) job_parent_resource_name_prefix -- is a prefix to filter the `job_parent_resource_name`. The latter is a path-like string that contains a hierarchy of resources. An example of `job_parent_resource_name` is: `/solvers/simcore%2Fservices%2Fcomp%2Fisolve/releases/1.3.4/jobs/f622946d-fd29-35b9-a193-abdd1095167c` @@ -109,8 +106,9 @@ async def list_projects_marked_as_jobs( project_to_groups.c.gid.in_(sa.select(user_groups_query.c.gid)), project_to_groups.c.read.is_(True), projects.c.workspace_id.is_( + # ONLY projects in private workspaces None - ), # ONLY projects in private workspaces + ), ) ) From 45b8767a06f59f52c7012df790f971f46f8c5532 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:32:20 +0200 Subject: [PATCH 44/48] @bisgaard-itis review: url jobs moved --- .../api/routes/solvers_jobs_getters.py | 35 +++++++++++++++++-- .../models/schemas/jobs.py | 32 +---------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py index 38015715c37..8c4bea421b8 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs_getters.py @@ -14,7 +14,7 @@ from models_library.projects_nodes_io import BaseFileLink from models_library.users import UserID from models_library.wallets import ZERO_CREDITS -from pydantic import NonNegativeInt +from pydantic import HttpUrl, NonNegativeInt from pydantic.types import PositiveInt from servicelib.fastapi.requests_decorators import cancel_on_disconnect from servicelib.logging_utils import log_context @@ -36,7 +36,6 @@ JobLog, JobMetadata, JobOutputs, - update_job_urls, ) from ...models.schemas.model_adapter import ( PricingUnitGetLegacy, @@ -119,6 +118,36 @@ } +def _update_job_urls( + job: Job, + solver_key: SolverKeyId, + solver_version: VersionStr, + job_id: JobID | str, + url_for: Callable[..., HttpUrl], +) -> Job: + job.url = url_for( + "get_job", + solver_key=solver_key, + version=solver_version, + job_id=job_id, + ) + + job.runner_url = url_for( + "get_solver_release", + solver_key=solver_key, + version=solver_version, + ) + + job.outputs_url = url_for( + "get_job_outputs", + solver_key=solver_key, + version=solver_version, + job_id=job_id, + ) + + return job + + router = APIRouter() @@ -150,7 +179,7 @@ async def list_all_solvers_jobs( for job in jobs: solver_key, version, job_id = parse_resources_ids(job.resource_name) - update_job_urls(job, solver_key, version, job_id, url_for) + _update_job_urls(job, solver_key, version, job_id, url_for) return create_page( jobs, diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py index 747e6095b16..86abb0a8741 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/jobs.py @@ -48,7 +48,7 @@ # - custom metadata # from .programs import Program, ProgramKeyId -from .solvers import Solver, SolverKeyId +from .solvers import Solver JobID: TypeAlias = UUID @@ -362,36 +362,6 @@ def get_outputs_url( return None -def update_job_urls( - job: Job, - solver_key: SolverKeyId, - solver_version: VersionStr, - job_id: JobID | str, - url_for: Callable[..., HttpUrl], -) -> Job: - job.url = url_for( - "get_job", - solver_key=solver_key, - version=solver_version, - job_id=job_id, - ) - - job.runner_url = url_for( - "get_solver_release", - solver_key=solver_key, - version=solver_version, - ) - - job.outputs_url = url_for( - "get_job_outputs", - solver_key=solver_key, - version=solver_version, - job_id=job_id, - ) - - return job - - PercentageInt: TypeAlias = Annotated[int, Field(ge=0, le=100)] From e8b8996b249f73767a6539736eab50cd3804fe19 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:35:56 +0200 Subject: [PATCH 45/48] @GitHK review: exception --- .../src/simcore_service_api_server/_service_solvers.py | 6 ++++-- .../simcore_service_api_server/exceptions/custom_errors.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 759e99c85c4..73407d2e23b 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -12,6 +12,9 @@ from models_library.services_history import ServiceRelease from models_library.users import UserID from packaging.version import Version +from simcore_service_api_server.exceptions.custom_errors import ( + SolverServiceListJobsFiltersError, +) from .models.api_resources import compose_resource_name from .models.schemas.jobs import Job, JobInputs @@ -110,8 +113,7 @@ async def list_jobs( collection_or_resource_ids.append("releases") collection_or_resource_ids.append(solver_version) elif solver_version: - msg = "solver_version is set but solver_id is not. Please provide both or none of them" - raise ValueError(msg) + raise SolverServiceListJobsFiltersError job_parent_resource_name_prefix = compose_resource_name( *collection_or_resource_ids diff --git a/services/api-server/src/simcore_service_api_server/exceptions/custom_errors.py b/services/api-server/src/simcore_service_api_server/exceptions/custom_errors.py index 17f22d16fae..4157f23eee4 100644 --- a/services/api-server/src/simcore_service_api_server/exceptions/custom_errors.py +++ b/services/api-server/src/simcore_service_api_server/exceptions/custom_errors.py @@ -16,3 +16,7 @@ class MissingWalletError(CustomBaseError): class ApplicationSetupError(CustomBaseError): pass + + +class SolverServiceListJobsFiltersError(CustomBaseError, ValueError): + msg_template = "solver_version is set but solver_id is not. Please provide both or none of them" From ebe4b87e374255ada80311588453d30ccd66be62 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:38:55 +0200 Subject: [PATCH 46/48] @sanderegg review:doc --- .../src/simcore_service_api_server/models/schemas/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py index fb2fd0e6a25..07144ba5b76 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/_base.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/_base.py @@ -45,7 +45,9 @@ class BaseService(BaseModel): ] description: Annotated[ str | None, - StringConstraints( # NOTE: keep StringConstraints before to keep the openapi schema + StringConstraints( + # NOTE: Place `StringConstraints` before `trim_string_before` for valid OpenAPI schema due to a Pydantic limitation. + # SEE `test_trim_string_before_with_string_constraints` max_length=1000 ), trim_string_before(max_length=1000), From bd9542238ef8d09698cbf9f880ef02d4afeb67ac Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 15:05:14 +0200 Subject: [PATCH 47/48] fixes dependencies and release notes --- .../api/routes/programs.py | 12 ++++++++---- .../simcore_service_api_server/api/routes/solvers.py | 8 ++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/programs.py b/services/api-server/src/simcore_service_api_server/api/routes/programs.py index 849e6692c9c..2610aa4074c 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/programs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/programs.py @@ -23,7 +23,7 @@ from ..._service_programs import ProgramService from ...api.routes._constants import ( DEFAULT_MAX_STRING_LENGTH, - FMSG_CHANGELOG_ADDED_IN_VERSION, + FMSG_CHANGELOG_NEW_IN_VERSION, create_route_description, ) from ...models.basic_types import VersionStr @@ -41,7 +41,9 @@ response_model=Page[Program], description=create_route_description( base="Lists the latest of all available programs", - changelog=[FMSG_CHANGELOG_ADDED_IN_VERSION.format("0.8")], + changelog=[ + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.8"), + ], ), include_in_schema=False, # TO BE RELEASED in 0.8 ) @@ -77,7 +79,10 @@ async def list_programs( "/{program_key:path}/releases", response_model=Page[Program], description=create_route_description( - changelog=[FMSG_CHANGELOG_ADDED_IN_VERSION.format("0.8")], + base="Lists the latest of all available programs", + changelog=[ + FMSG_CHANGELOG_NEW_IN_VERSION.format("0.8"), + ], ), include_in_schema=False, # TO BE RELEASED in 0.8 ) @@ -89,7 +94,6 @@ async def list_program_history( product_name: Annotated[str, Depends(get_product_name)], page_params: Annotated[PaginationParams, Depends()], ): - """Lists the latest of all available programs""" programs, page_meta = await program_service.list_program_history( program_key=program_key, user_id=user_id, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 9419632daf6..b9d7cb015d9 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -110,7 +110,7 @@ async def get_solvers_page( page_params: Annotated[PaginationParams, Depends()], user_id: Annotated[int, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): solvers, page_meta = await solver_service.latest_solvers( @@ -150,7 +150,7 @@ async def get_solvers_page( ) async def list_solvers_releases( user_id: Annotated[int, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): @@ -230,7 +230,7 @@ async def get_solver( async def list_solver_releases( solver_key: SolverKeyId, user_id: Annotated[int, Depends(get_current_user_id)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], product_name: Annotated[str, Depends(get_product_name)], ): @@ -275,7 +275,7 @@ async def get_solver_releases_page( user_id: Annotated[int, Depends(get_current_user_id)], product_name: Annotated[str, Depends(get_product_name)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], - solver_service: Annotated[SolverService, Depends(SolverService)], + solver_service: Annotated[SolverService, Depends(get_solver_service)], ): solvers, page_meta = await solver_service.solver_release_history( user_id=user_id, From 42a344e5645d9ad373011e502a17044488a1319c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 29 Apr 2025 15:05:30 +0200 Subject: [PATCH 48/48] reverted OAS --- services/api-server/openapi.json | 385 ------------------------------- 1 file changed, 385 deletions(-) diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 7b5250e3f9c..d8e32bbd473 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -1256,140 +1256,6 @@ } } }, - "/v0/programs": { - "get": { - "tags": [ - "programs" - ], - "summary": "List Programs", - "description": "Lists the latest of all available programs", - "operationId": "list_programs", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 50, - "minimum": 1, - "default": 20, - "title": "Limit" - } - }, - { - "name": "offset", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "default": 0, - "title": "Offset" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Program_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, - "/v0/programs/{program_key}/releases": { - "get": { - "tags": [ - "programs" - ], - "summary": "List Program History", - "description": "Lists the latest of all available programs", - "operationId": "list_program_history", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "program_key", - "in": "path", - "required": true, - "schema": { - "type": "string", - "pattern": "^simcore/services/dynamic/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", - "title": "Program Key" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 50, - "minimum": 1, - "default": 20, - "title": "Limit" - } - }, - { - "name": "offset", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "default": 0, - "title": "Offset" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Program_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/programs/{program_key}/releases/{version}": { "get": { "tags": [ @@ -1642,68 +1508,6 @@ ] } }, - "/v0/solvers/page": { - "get": { - "tags": [ - "solvers" - ], - "summary": "Get Solvers Page", - "description": "Lists all available solvers (latest version) with pagination", - "operationId": "get_solvers_page", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 50, - "minimum": 1, - "default": 20, - "title": "Limit" - } - }, - { - "name": "offset", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "default": 0, - "title": "Offset" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Solver_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/solvers/releases": { "get": { "tags": [ @@ -2017,77 +1821,6 @@ } } }, - "/v0/solvers/{solver_key}/releases/page": { - "get": { - "tags": [ - "solvers" - ], - "summary": "Get Solver Releases Page", - "operationId": "get_solver_releases_page", - "security": [ - { - "HTTPBasic": [] - } - ], - "parameters": [ - { - "name": "solver_key", - "in": "path", - "required": true, - "schema": { - "type": "string", - "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", - "title": "Solver Key" - } - }, - { - "name": "limit", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "maximum": 50, - "minimum": 1, - "default": 20, - "title": "Limit" - } - }, - { - "name": "offset", - "in": "query", - "required": false, - "schema": { - "type": "integer", - "minimum": 0, - "default": 0, - "title": "Offset" - } - } - ], - "responses": { - "200": { - "description": "Successful Response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Page_Solver_" - } - } - } - }, - "422": { - "description": "Validation Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - } - } - } - } - }, "/v0/solvers/{solver_key}/releases/{version}": { "get": { "tags": [ @@ -7750,124 +7483,6 @@ ], "title": "Page[LicensedItemGet]" }, - "Page_Program_": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/Program" - }, - "type": "array", - "title": "Items" - }, - "total": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "null" - } - ], - "title": "Total" - }, - "limit": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "null" - } - ], - "title": "Limit" - }, - "offset": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "null" - } - ], - "title": "Offset" - }, - "links": { - "$ref": "#/components/schemas/Links" - } - }, - "type": "object", - "required": [ - "items", - "total", - "limit", - "offset", - "links" - ], - "title": "Page[Program]" - }, - "Page_Solver_": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/Solver" - }, - "type": "array", - "title": "Items" - }, - "total": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "null" - } - ], - "title": "Total" - }, - "limit": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "null" - } - ], - "title": "Limit" - }, - "offset": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "null" - } - ], - "title": "Offset" - }, - "links": { - "$ref": "#/components/schemas/Links" - } - }, - "type": "object", - "required": [ - "items", - "total", - "limit", - "offset", - "links" - ], - "title": "Page[Solver]" - }, "Page_Study_": { "properties": { "items": {