diff --git a/pro_tes/app.py b/pro_tes/app.py index c2ade93..85c3795 100644 --- a/pro_tes/app.py +++ b/pro_tes/app.py @@ -16,6 +16,7 @@ def init_app() -> FlaskApp: """ foca = Foca( config_file=Path(__file__).resolve().parent / "config.yaml", + custom_config_model="pro_tes.config_models.CustomConfig", ) app = foca.create_app() with app.app.app_context(): diff --git a/pro_tes/config.yaml b/pro_tes/config.yaml index 78ea53c..99f8185 100644 --- a/pro_tes/config.yaml +++ b/pro_tes/config.yaml @@ -106,44 +106,46 @@ exceptions: status_member: ["code"] exceptions: pro_tes.exceptions.exceptions -controllers: - post_task: - db: - insert_attempts: 10 - task_id: - charset: string.ascii_uppercase + string.digits - length: 6 - timeout: - post: null - poll: 2 - job: null - polling: - wait: 3 - attempts: 100 - list_tasks: - default_page_size: 5 +# Custom configuration +custom: + controllers: + post_task: + db: + insert_attempts: 10 + task_id: + charset: string.ascii_uppercase + string.digits + length: 6 + timeout: + post: null + poll: 2 + job: null + polling: + wait: 3 + attempts: 100 + list_tasks: + default_page_size: 5 celery: monitor: timeout: 0.1 message_maxsize: 16777216 -serviceInfo: - doc: Proxy TES for distributing tasks across a list of service TES instances - name: proTES - storage: - - file:///path/to/local/storage + serviceInfo: + doc: Proxy TES for distributing tasks across a list of service TES instances + name: proTES + storage: + - file:///path/to/local/storage -tes: - service_list: - - "https://csc-tesk-noauth.rahtiapp.fi" - - "https://funnel.cloud.e-infra.cz/" - - "https://tesk-eu.hypatia-comp.athenarc.gr" - - "https://tesk-na.cloud.e-infra.cz" - - "https://vm4816.kaj.pouta.csc.fi/" + tes: + service_list: + - "https://csc-tesk-noauth.rahtiapp.fi" + - "https://funnel.cloud.e-infra.cz/" + - "https://tesk-eu.hypatia-comp.athenarc.gr" + - "https://tesk-na.cloud.e-infra.cz" + - "https://vm4816.kaj.pouta.csc.fi/" -storeLogs: - execution_trace: True + storeLogs: + execution_trace: True -middlewares: - - - "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance" - - "pro_tes.plugins.middlewares.task_distribution.random.TaskDistributionRandom" + middlewares: + - - "pro_tes.plugins.middlewares.task_distribution.distance.TaskDistributionDistance" + - "pro_tes.plugins.middlewares.task_distribution.random.TaskDistributionRandom" diff --git a/pro_tes/config_models.py b/pro_tes/config_models.py new file mode 100644 index 0000000..6f287c0 --- /dev/null +++ b/pro_tes/config_models.py @@ -0,0 +1,237 @@ +"""Custom app config models.""" + +from typing import List, Optional +import string +from pydantic import BaseModel # pylint: disable=no-name-in-module +from pro_tes.ga4gh.tes.models import Service as TesServiceInfo + + +# pragma pylint: disable=too-few-public-methods + + +class DB(BaseModel): + """DB config for post_task. + + Args: + insert_attempts: Number of attempts to insert a new task in DB. + + Attributes: + insert_attempts: Number of attempts to insert a new task in DB. + """ + + insert_attempts: int = 10 + + +class TaskID(BaseModel): + """Task ID config. + + Args: + charset: Characters to use when generating task IDs. + length: Length of the generated task ID. + + Attributes: + charset: Characters to use when generating task IDs. + length: Length of the generated task ID. + """ + + charset: str = string.ascii_uppercase + string.digits + length: int = 6 + + +class Timeout(BaseModel): + """Timeout config. + + Args: + post: Timeout for POST requests (None disables timeout). + poll: Timeout for polling. + job: Timeout for job execution (None disables timeout). + + Attributes: + post: Timeout for POST requests (None disables timeout). + poll: Timeout for polling. + job: Timeout for job execution (None disables timeout). + """ + + post: Optional[int] = None + poll: int = 2 + job: Optional[int] = None + + +class Polling(BaseModel): + """Polling config. + + Args: + wait: Wait time between polling attempts. + attempts: Max polling attempts before failure. + + Attributes: + wait: Wait time between polling attempts. + attempts: Max polling attempts before failure. + """ + + wait: int = 3 + attempts: int = 100 + + +class PostTask(BaseModel): + """Configuration for POST /task. + + Args: + db: DB insert behavior. + task_id: Task ID generation settings. + timeout: Timeout settings. + polling: Polling behavior. + + Attributes: + db: DB insert behavior. + task_id: Task ID generation settings. + timeout: Timeout settings. + polling: Polling behavior. + """ + + db: DB = DB() + task_id: TaskID = TaskID() + timeout: Timeout = Timeout() + polling: Polling = Polling() + + +class ListTasks(BaseModel): + """Configuration for GET /tasks. + + Args: + default_page_size: Default pagination size. + + Attributes: + default_page_size: Default pagination size. + """ + + default_page_size: int = 5 + + +class Monitor(BaseModel): + """Celery monitor settings. + + Args: + timeout: Timeout to wait for Celery monitoring. + + Attributes: + timeout: Timeout to wait for Celery monitoring. + """ + + timeout: float = 0.1 + + +class Celery(BaseModel): + """Celery configuration. + + Args: + monitor: Monitor settings. + message_maxsize: Maximum allowed message size. + + Attributes: + monitor: Monitor settings. + message_maxsize: Maximum allowed message size. + """ + + monitor: Monitor = Monitor() + message_maxsize: int = 16777216 + + +class Controllers(BaseModel): + """Controller configurations. + + Args: + post_task: Settings for POST /task. + list_tasks: Settings for GET /tasks. + celery: Celery background task settings. + + Attributes: + post_task: Settings for POST /task. + list_tasks: Settings for GET /tasks. + celery: Celery background task settings. + """ + + post_task: PostTask = PostTask() + list_tasks: ListTasks = ListTasks() + celery: Celery = Celery() + + +class Tes(BaseModel): + """TES backend configuration. + + Args: + service_list: List of available TES services. + + Attributes: + service_list: List of available TES services. + """ + + service_list: List[str] = [ + "https://csc-tesk-noauth.rahtiapp.fi", + "https://funnel.cloud.e-infra.cz/", + "https://tesk-eu.hypatia-comp.athenarc.gr", + "https://tesk-na.cloud.e-infra.cz", + "https://vm4816.kaj.pouta.csc.fi/", + ] + + +class StoreLogs(BaseModel): + """Logging configuration. + + Args: + execution_trace: Whether to store execution trace logs. + + Attributes: + execution_trace: Whether to store execution trace logs. + """ + + execution_trace: bool = True + + +class Middlewares(BaseModel): + """Middleware configuration. + + Args: + __root__: A list of middleware class paths. + + Attributes: + __root__: A list of middleware class paths. + """ + + __root__: List[List[str]] = [ + [ + ( + "pro_tes.plugins.middlewares.task_distribution.distance." + "TaskDistributionDistance" + ), + ( + "pro_tes.plugins.middlewares.task_distribution.random." + "TaskDistributionRandom" + ), + ] + ] + + +class CustomConfig(BaseModel): + """Custom app configuration. + + Args: + controllers: All controller-related config. + tes: TES service list and defaults. + store_logs: Logging preferences. + middlewares: Middleware class paths. + service_info: Metadata about the service. + + Attributes: + controllers: All controller-related config. + tes: TES service list and defaults. + store_logs: Logging preferences. + middlewares: Middleware class paths. + service_info: Metadata about the service. + """ + + controllers: Controllers = Controllers() + tes: Tes = Tes() + storeLogs: StoreLogs = StoreLogs() + middlewares: Middlewares = Middlewares() + service_info: TesServiceInfo diff --git a/pro_tes/ga4gh/tes/service_info.py b/pro_tes/ga4gh/tes/service_info.py index f9a24c8..e2e3859 100644 --- a/pro_tes/ga4gh/tes/service_info.py +++ b/pro_tes/ga4gh/tes/service_info.py @@ -65,7 +65,9 @@ def init_service_info_from_config(self) -> None: Set service info only if it does not yet exist. """ - service_info_conf = current_app.config.foca.serviceInfo # type: ignore + service_info_conf = ( + current_app.config.foca.custom.serviceInfo # type: ignore + ) try: service_info_db = self.get_service_info() except NotFound: diff --git a/pro_tes/ga4gh/tes/task_runs.py b/pro_tes/ga4gh/tes/task_runs.py index 55fa504..8367cd2 100644 --- a/pro_tes/ga4gh/tes/task_runs.py +++ b/pro_tes/ga4gh/tes/task_runs.py @@ -62,7 +62,7 @@ def __init__(self) -> None: self.db_client: Collection = ( self.foca_config.db.dbs["taskStore"].collections["tasks"].client ) - self.store_logs = self.foca_config.storeLogs["execution_trace"] + self.store_logs = self.foca_config.custom.storeLogs["execution_trace"] def create_task( # pylint: disable=too-many-statements,too-many-branches self, **kwargs @@ -86,7 +86,7 @@ def create_task( # pylint: disable=too-many-statements,too-many-branches # apply middlewares mw_handler = MiddlewareHandler() mw_handler.set_middlewares( - paths=current_app.config.foca.middlewares # type: ignore + paths=current_app.config.foca.custom.middlewares # type: ignore ) logger.debug(f"Middlewares registered: {mw_handler.middlewares}") request_modified = mw_handler.apply_middlewares(request=request) @@ -268,8 +268,10 @@ def list_tasks(self, **kwargs) -> dict: """ page_size = kwargs.get( "page_size", - self.foca_config.controllers["list_tasks"]["default_page_size"], - ) + self.foca_config.custom.controllers["list_tasks"][ + "default_page_size" + ], + ) page_token = kwargs.get("page_token") filter_dict = {} @@ -427,7 +429,7 @@ def _write_doc_to_db( Returns: Tuple of task id and worker id. """ - controller_config = self.foca_config.controllers["post_task"] + controller_config = self.foca_config.custom.controllers["post_task"] charset = controller_config["task_id"]["charset"] length = controller_config["task_id"]["length"] diff --git a/pro_tes/plugins/middlewares/task_distribution/base.py b/pro_tes/plugins/middlewares/task_distribution/base.py index 71f7541..a568f05 100644 --- a/pro_tes/plugins/middlewares/task_distribution/base.py +++ b/pro_tes/plugins/middlewares/task_distribution/base.py @@ -39,7 +39,9 @@ def apply_middleware(self, request: flask.Request) -> flask.Request: raise MiddlewareException("Request has no JSON payload.") self._set_tes_urls( tes_urls=deepcopy( - current_app.config.foca.tes["service_list"] # type: ignore + current_app.config.foca.custom.tes[ # type: ignore + "service_list" + ] ), request=request, ) diff --git a/pro_tes/tasks/track_task_progress.py b/pro_tes/tasks/track_task_progress.py index 156d2b9..88dcb7c 100644 --- a/pro_tes/tasks/track_task_progress.py +++ b/pro_tes/tasks/track_task_progress.py @@ -51,7 +51,7 @@ def task__track_task_progress( # pylint: disable=too-many-arguments,R0917 password: Password for basic authentication. """ foca_config: Config = current_app.config.foca # type: ignore - controller_config: dict = foca_config.controllers["post_task"] + controller_config: dict = foca_config.custom.controllers["post_task"] # create database client collection = _create_mongo_client(