Open
Description
When running a processing job that:
- uses a custom image
- takes some 'arguments'
- and being run in a local mode
..the user is forced to always pass an 'entrypoint' argument to the Processor, although the image itself might have a default ENTRYPOINT defined.
On the other hand, if the same configuration is run as a remote job, it works as expected - i.e. when an 'entrypoint' argument is not provided to the Processor, the job uses the one defined in the image itself.
To reproduce
One would need base/custom docker image with ENTRYPOINT defined.
See the snippet below:
from sagemaker.processing import Processor
processor = Processor(
image_uri='NNN.dkr.ecr.us-east-1.amazonaws.com/custom-image',
role="arn:aws:iam::NNN:role/service-role/some-role",
base_job_name="preprocess",
instance_count=1,
instance_type='local',
# entrypoint = ["python3", "preprocess.py"],
)
processor.run(
arguments=[
"--input-path", "my-input",
"--output-path", "my-output"
]
)The code above, with the line entrypoint = ["python3", "preprocess.py"] commented out, fails with the following exception:
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/image.py:757, in _SageMakerContainer._create_docker_host(self, host, environment, optml_subdirs, command, volumes)
755 host_config["entrypoint"] = self.container_entrypoint
756 if self.container_arguments:
--> 757 host_config["entrypoint"] = host_config["entrypoint"] + self.container_arguments
759 # for GPU support pass in nvidia as the runtime, this is equivalent
760 # to setting --runtime=nvidia in the docker commandline.
761 if self.instance_type == "local_gpu":
KeyError: 'entrypoint'
Expected behavior
The job should not need an 'entrypoint' argument specified, it should default to the entrypoint defined inside the docker image.
Screenshots or logs
Full exception stack:
INFO:sagemaker:Creating processing-job with name preprocess-2022-10-18-09-46-41-207
INFO:sagemaker.local.local_session:Starting processing job
INFO:botocore.credentials:Found credentials in shared credentials file: ~/.aws/credentials
Job Name: preprocess-2022-10-18-09-46-41-207
Inputs: []
Outputs: []
WARNING:sagemaker.local.image:Using the short-lived AWS credentials found in session. They might expire while running.
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
Cell In [4], line 12
1 from sagemaker.processing import Processor
3 processor = Processor(
4 image_uri='451191978663.dkr.ecr.us-east-1.amazonaws.com/a204311-ml-workspace-modelopsextehvui-use1:test_experiment_123-preprocess',
5 role="arn:aws:iam::451191978663:role/service-role/a204311-ml-workspace-ModelOpsExtEhVUI-prod-use1",
(...)
9 # entrypoint = ["python3", "preprocess.py"],
10 )
---> 12 processor.run(
13 arguments=[
14 "--input-path", "my-input",
15 "--output-path", "my-output"
16 ]
17 )
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/workflow/pipeline_context.py:248, in runnable_by_pipeline.<locals>.wrapper(*args, **kwargs)
245 run_func(*args, **kwargs)
246 return self_instance.sagemaker_session.context
--> 248 return run_func(*args, **kwargs)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/processing.py:203, in Processor.run(self, inputs, outputs, arguments, wait, logs, job_name, experiment_config, kms_key)
190 raise ValueError(
191 """Logs can only be shown if wait is set to True.
192 Please either set wait to True or set logs to False."""
193 )
195 normalized_inputs, normalized_outputs = self._normalize_args(
196 job_name=job_name,
197 arguments=arguments,
(...)
200 outputs=outputs,
201 )
--> 203 self.latest_job = ProcessingJob.start_new(
204 processor=self,
205 inputs=normalized_inputs,
206 outputs=normalized_outputs,
207 experiment_config=experiment_config,
208 )
209 self.jobs.append(self.latest_job)
210 if wait:
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/processing.py:796, in ProcessingJob.start_new(cls, processor, inputs, outputs, experiment_config)
793 print("Outputs: ", process_args["output_config"]["Outputs"])
795 # Call sagemaker_session.process using the arguments dictionary.
--> 796 processor.sagemaker_session.process(**process_args)
798 return cls(
799 processor.sagemaker_session,
800 processor._current_job_name,
(...)
803 processor.output_kms_key,
804 )
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/session.py:956, in Session.process(self, inputs, output_config, job_name, resources, stopping_condition, app_specification, environment, network_config, role_arn, tags, experiment_config)
953 LOGGER.debug("process request: %s", json.dumps(request, indent=4))
954 self.sagemaker_client.create_processing_job(**request)
--> 956 self._intercept_create_request(process_request, submit, self.process.__name__)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/session.py:4317, in Session._intercept_create_request(self, request, create, func_name)
4304 def _intercept_create_request(
4305 self, request: typing.Dict, create, func_name: str = None # pylint: disable=unused-argument
4306 ):
4307 """This function intercepts the create job request.
4308
4309 PipelineSession inherits this Session class and will override
(...)
4315 func_name (str): the name of the function needed intercepting
4316 """
-> 4317 return create(request)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/session.py:954, in Session.process.<locals>.submit(request)
952 LOGGER.info("Creating processing-job with name %s", job_name)
953 LOGGER.debug("process request: %s", json.dumps(request, indent=4))
--> 954 self.sagemaker_client.create_processing_job(**request)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/local_session.py:128, in LocalSagemakerClient.create_processing_job(self, ProcessingJobName, AppSpecification, ProcessingResources, Environment, ProcessingInputs, ProcessingOutputConfig, **kwargs)
126 processing_job = _LocalProcessingJob(container)
127 logger.info("Starting processing job")
--> 128 processing_job.start(
129 ProcessingInputs, ProcessingOutputConfig, Environment, ProcessingJobName
130 )
132 LocalSagemakerClient._processing_jobs[ProcessingJobName] = processing_job
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/entities.py:140, in _LocalProcessingJob.start(self, processing_inputs, processing_output_config, environment, processing_job_name)
137 self.processing_output_config = processing_output_config
138 self.environment = environment
--> 140 self.container.process(
141 processing_inputs, processing_output_config, environment, processing_job_name
142 )
144 self.end_time = datetime.datetime.now()
145 self.state = self._COMPLETED
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/image.py:153, in _SageMakerContainer.process(self, processing_inputs, processing_output_config, environment, processing_job_name)
144 _create_processing_config_file_directories(self.container_root, host)
145 self.write_processing_config_files(
146 host,
147 environment,
(...)
150 processing_job_name,
151 )
--> 153 self._generate_compose_file(
154 "process", additional_volumes=volumes, additional_env_vars=environment
155 )
156 compose_command = self._compose()
158 if _ecr_login_if_needed(self.sagemaker_session.boto_session, self.image):
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/image.py:671, in _SageMakerContainer._generate_compose_file(self, command, additional_volumes, additional_env_vars)
668 elif command == "process":
669 optml_dirs = {"output", "config"}
--> 671 services = {
672 h: self._create_docker_host(h, environment, optml_dirs, command, additional_volumes)
673 for h in self.hosts
674 }
676 content = {
677 # Use version 2.3 as a minimum so that we can specify the runtime
678 "version": "2.3",
679 "services": services,
680 "networks": {"sagemaker-local": {"name": "sagemaker-local"}},
681 }
683 docker_compose_path = os.path.join(self.container_root, DOCKER_COMPOSE_FILENAME)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/image.py:672, in <dictcomp>(.0)
668 elif command == "process":
669 optml_dirs = {"output", "config"}
671 services = {
--> 672 h: self._create_docker_host(h, environment, optml_dirs, command, additional_volumes)
673 for h in self.hosts
674 }
676 content = {
677 # Use version 2.3 as a minimum so that we can specify the runtime
678 "version": "2.3",
679 "services": services,
680 "networks": {"sagemaker-local": {"name": "sagemaker-local"}},
681 }
683 docker_compose_path = os.path.join(self.container_root, DOCKER_COMPOSE_FILENAME)
File ~/work/temp/labs-test_project/.venv/lib/python3.8/site-packages/sagemaker/local/image.py:757, in _SageMakerContainer._create_docker_host(self, host, environment, optml_subdirs, command, volumes)
755 host_config["entrypoint"] = self.container_entrypoint
756 if self.container_arguments:
--> 757 host_config["entrypoint"] = host_config["entrypoint"] + self.container_arguments
759 # for GPU support pass in nvidia as the runtime, this is equivalent
760 # to setting --runtime=nvidia in the docker commandline.
761 if self.instance_type == "local_gpu":
KeyError: 'entrypoint'
System information
A description of your system. Please provide:
- SageMaker Python SDK version: 2.112.2
- Framework name (eg. PyTorch) or algorithm (eg. KMeans): Processor
- Framework version: -
- Python version: 3.8
- CPU or GPU: -
- Custom Docker image (Y/N): Y