Clarify throws TypeError: can't pickle SSLContext objects

[dhawalkp]

Describe the bug
clarify_processor.run_explainability throws TypeError: can't pickle SSLContext objects

To reproduce

1. Create a sagemaker.Model object
2. Configure model_config = clarify.ModelConfig to point to this model
3. run clarify_processor.run_explainability
   Throws above error

Expected behavior
Should not throw any error

Screenshots or logs

---

TypeError Traceback (most recent call last)
in
5 data_config=explainability_data_config,
6 model_config=model_config,
----> 7 explainability_config=shap_config,
8 )

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/clarify.py in run_explainability(self, data_config, model_config, explainability_config, model_scores, wait, logs, job_name, kms_key, experiment_config)
1128 """
1129 analysis_config = data_config.get_config()
-> 1130 predictor_config = model_config.get_predictor_config()
1131 if isinstance(model_scores, ModelPredictedLabelConfig):
1132 (

~/anaconda3/envs/python3/lib/python3.6/site-packages/sagemaker/clarify.py in get_predictor_config(self)
280 def get_predictor_config(self):
281 """Returns part of the predictor dictionary of the analysis config."""
--> 282 return copy.deepcopy(self.predictor_config)
283
284

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_method(x, memo)
245
246 def _deepcopy_method(x, memo): # Copy instance methods
--> 247 return type(x)(x.func, deepcopy(x.self, memo))
248 d[types.MethodType] = _deepcopy_method
249

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
178 y = x
179 else:
--> 180 y = _reconstruct(x, memo, *rv)
181
182 # If is its own copy, don't memoize.

~/anaconda3/envs/python3/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
278 if state is not None:
279 if deep:
--> 280 state = deepcopy(state, memo)
281 if hasattr(y, 'setstate'):
282 y.setstate(state)

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
148 copier = _deepcopy_dispatch.get(cls)
149 if copier:
--> 150 y = copier(x, memo)
151 else:
152 try:

~/anaconda3/envs/python3/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
238 memo[id(x)] = y
239 for key, value in x.items():
--> 240 y[deepcopy(key, memo)] = deepcopy(value, memo)
241 return y
242 d[dict] = _deepcopy_dict

~/anaconda3/envs/python3/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
167 reductor = getattr(x, "reduce_ex", None)
168 if reductor:
--> 169 rv = reductor(4)
170 else:
171 reductor = getattr(x, "reduce", None)

TypeError: can't pickle SSLContext objects

System information
A description of your system. Please provide:

• SageMaker Python SDK version:2.76.0
• Framework name (eg. PyTorch) or algorithm (eg. KMeans):BYOC
• Framework version: BYOC
• Python version: 3.6
• CPU or GPU: CPU
• Custom Docker image (Y/N): Y

Additional context
Do not use deepcopy to get the dict object for model config at below line. This should fix the issue.

def get_predictor_config(self):
281 """Returns part of the predictor dictionary of the analysis config."""
--> 282 return copy.deepcopy(self.predictor_config)

[xgchena]

Thanks for reporting the issue.

I think the problem is that you passed the Model object instead of the model name to the ModelConfig object.

Assume that model is the Model object, then this should work

model_config = clarify.ModelConfig(
    model_name=model.name,  # The model_name parameter expects the model name string.
    ...
)

The docstring of the model_name parameter looks confusing, we will fix it.

model_name (str) – Model name (as created by ‘CreateModel’).

[xgchena]

Cross-reference: internal tracking task https://tiny.amazon.com/vu2katay