feat(decorators): primary region fail-over

README.md

@@ -95,6 +95,35 @@ cache = SecretCache(config=cache_config, client=client)
 secret = cache.get_secret_string('mysecret')
 ```
 
+#### Cross-account access
+In a case when Secrets Manager isolated in a different account, the client can use cross-account access with assumed role as following:
+```python
+from aws_secretsmanager_caching import SecretCache
+
+def assume_account_role(account_id, role_name, duration=900):
+    client = botocore.session.get_session().create_client("sts")
+    role_arn = f"arn:aws:iam::{account_id}:role/{role_name}"
+    session_name = "SecretCacheSession"
+    response = client.assume_role(
+        RoleArn=role_arn,
+        RoleSessionName=session_name,
+        DurationSeconds=duration
+    )
+    session = botocore.session.Session()
+    session.set_credentials(
+        access_key=response["Credentials"]["AccessKeyId"],
+        secret_key=response["Credentials"]["SecretAccessKey"],
+        token=response["Credentials"]["SessionToken"]
+    )
+    return session
+
+session = assume_account_role(account_id="0123456789012", role_name="RoleToAssume", duration=3600)
+client = session.create_client('secretsmanager')
+cache = SecretCache(config=cache_config, client=client)
+
+secret = cache.get_secret_string('mysecret')
+```
+
 #### Cache Configuration
 You can configure the cache config object with the following parameters:
 * `max_cache_size` - The maximum number of secrets to cache.  The default value is `1024`.
@@ -126,6 +155,25 @@ def function_to_be_decorated(arg1, arg2, arg3):
     # arg2 and arg3, in this example, must still be passed when calling function_to_be_decorated().
 ```
 
+
+##### Multi-region outage failover
+Both decorators also accepts a cache list for a multi-region outage failover case (this requires enabled multi-region secrets replication).
+
+```python
+from aws_secretsmanager_caching import InjectKeywordedSecretString
+
+
+regions = ["us-west-2", "us-east-1", "us-east-2"]
+session = assume_account_role(account_id="0123456789012", role_name="RoleToAssume")
+clients = (session.create_client("secretsmanager", region) for region in regions)
+replica = [SecretCache(client=client) for client in clients]
+
+
+@InjectSecretString(secret_id='mysimplesecret', cache=replica[0], caches=replica[1:])
+def function_to_be_decorated(secret, arg2, arg3):
+    pass
+```
+
 ## Getting Help
 Please use these community resources for getting help:
 * Ask a question on [Stack Overflow](https://stackoverflow.com/) and tag it with [aws-secrets-manager](https://stackoverflow.com/questions/tagged/aws-secrets-manager).

src/aws_secretsmanager_caching/decorators.py

@@ -13,11 +13,15 @@
 """Decorators for use with caching library """
 import json
 
+from abc import ABC
+
+import botocore.exceptions
 
-class InjectSecretString:
-    """Decorator implementing high-level Secrets Manager caching client"""
 
-    def __init__(self, secret_id, cache):
+class InjectSecretAbstract(ABC):
+    """High-level abstraction for Secrets Manager decorators."""
+
+    def __init__(self, secret_id, cache, caches=None):
         """
         Constructs a decorator to inject a single non-keyworded argument from a cached secret for a given function.
 
@@ -26,11 +30,46 @@ def __init__(self, secret_id, cache):
 
         :type cache: aws_secretsmanager_caching.SecretCache
         :param cache: Secret cache
+
+        :type cache: Optional[List[aws_secretsmanager_caching.SecretCache]]
+        :param cache: Multiple additional secret caches for multiregion failover
         """
 
-        self.cache = cache
+        self.cache_id = 0
+        self.caches = [cache]
+        if caches:
+            self.caches.extend(caches)
         self.secret_id = secret_id
 
+    def _get_cached_secret(self):
+        """
+        Return cached secret.
+
+        :type cache: Union[Dict, str]
+        :param cache: Plaintext secret value or object
+        """
+
+        n_caches = len(self.caches)
+        # Probe each replica (including primary) starting with the current one
+        replicas = [i % n_caches for i in range(self.cache_id, self.cache_id + n_caches)]
+        for replica in replicas:
+            try:
+                secret = self.caches[replica].get_secret_string(secret_id=self.secret_id)
+                self.cache_id = replica
+                break
+            except botocore.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] in {"InternalFailure", "ServiceUnavailable"}:
+                    if replica == replicas[-1]:
+                        # All possible replicas were probed
+                        raise
+                else:
+                    raise
+        return secret
+
+
+class InjectSecretString(InjectSecretAbstract):
+    """Decorator implementing high-level Secrets Manager caching client"""
+
     def __call__(self, func):
         """
         Return a function with cached secret injected as first argument.
@@ -39,8 +78,7 @@ def __call__(self, func):
         :param func: The function for injecting a single non-keyworded argument too.
         :return The function with the injected argument.
         """
-
-        secret = self.cache.get_secret_string(secret_id=self.secret_id)
+        secret = self._get_cached_secret()
 
         def _wrapped_func(*args, **kwargs):
             """
@@ -51,10 +89,10 @@ def _wrapped_func(*args, **kwargs):
         return _wrapped_func
 
 
-class InjectKeywordedSecretString:
+class InjectKeywordedSecretString(InjectSecretAbstract):
     """Decorator implementing high-level Secrets Manager caching client using JSON-based secrets"""
 
-    def __init__(self, secret_id, cache, **kwargs):
+    def __init__(self, secret_id, cache, caches=None, **kwargs):
         """
         Construct a decorator to inject a variable list of keyword arguments to a given function with resolved values
         from a cached secret.
@@ -67,11 +105,13 @@ def __init__(self, secret_id, cache, **kwargs):
 
         :type cache: aws_secretsmanager_caching.SecretCache
         :param cache: Secret cache
+
+        :type cache: Optional[List[aws_secretsmanager_caching.SecretCache]]
+        :param cache: Multiple secret caches for multiregion failover
         """
 
-        self.cache = cache
+        super().__init__(secret_id, cache, caches)
         self.kwarg_map = kwargs
-        self.secret_id = secret_id
 
     def __call__(self, func):
         """
@@ -83,7 +123,8 @@ def __call__(self, func):
         """
 
         try:
-            secret = json.loads(self.cache.get_secret_string(secret_id=self.secret_id))
+            secret = self._get_cached_secret()
+            secret = json.loads(secret)
         except json.decoder.JSONDecodeError:
             raise RuntimeError('Cached secret is not valid JSON') from None
 

test/unit/test_decorators.py

@@ -24,8 +24,8 @@
 
 class TestAwsSecretsManagerCachingInjectKeywordedSecretStringDecorator(unittest.TestCase):
 
-    def get_client(self, response={}, versions=None, version_response=None):
-        client = botocore.session.get_session().create_client('secretsmanager', region_name='us-west-2')
+    def get_client(self, response={}, versions=None, version_response=None, region_name='us-west-2', error=False):
+        client = botocore.session.get_session().create_client('secretsmanager', region_name=region_name)
         stubber = Stubber(client)
         expected_params = {'SecretId': 'test'}
         if versions:
@@ -142,18 +142,45 @@ def function_to_be_decorated(func_username, func_password, keyworded_argument='f
 
             function_to_be_decorated()
 
+    def test_region_failure(self):
+        secret = {
+            'username': 'secret_username',
+            'password': 'secret_password'
+        }
+
+        secret_string = json.dumps(secret)
+
+        response = {}
+        versions = {
+            '01234567890123456789012345678901': ['AWSCURRENT']
+        }
+        version_response = {'SecretString': secret_string}
+        cache = SecretCache(client=self.get_client(response, versions, version_response, error=True))
+        caches = [SecretCache(client=self.get_client(response, versions, version_response, region_name='us-east-1'))]
+        @InjectKeywordedSecretString(secret_id='test', cache=cache, caches=caches, func_username='username', func_password='password')
+        def function_to_be_decorated(func_username, func_password, keyworded_argument='foo'):
+            self.assertEqual(secret['username'], func_username)
+            self.assertEqual(secret['password'], func_password)
+            self.assertEqual(keyworded_argument, 'foo')
+            return 'OK'
+
+        self.assertEqual(function_to_be_decorated(), 'OK')
+
 
 class TestAwsSecretsManagerCachingInjectSecretStringDecorator(unittest.TestCase):
 
-    def get_client(self, response={}, versions=None, version_response=None):
-        client = botocore.session.get_session().create_client('secretsmanager', region_name='us-west-2')
+    def get_client(self, response={}, versions=None, version_response=None, region_name='us-west-2', error=False):
+        client = botocore.session.get_session().create_client('secretsmanager', region_name=region_name)
         stubber = Stubber(client)
         expected_params = {'SecretId': 'test'}
-        if versions:
-            response['VersionIdsToStages'] = versions
-        stubber.add_response('describe_secret', response, expected_params)
-        if version_response is not None:
-            stubber.add_response('get_secret_value', version_response)
+        if error:
+            stubber.add_client_error('describe_secret', 'InternalFailure', http_status_code=502)
+        else:
+            if versions:
+                response['VersionIdsToStages'] = versions
+            stubber.add_response('describe_secret', response, expected_params)
+            if version_response is not None:
+                stubber.add_response('get_secret_value', version_response)
         stubber.activate()
         return client
 
@@ -191,3 +218,22 @@ def function_to_be_decorated(arg1, arg2, arg3):
             self.assertEqual(arg3, 'bar')
 
         function_to_be_decorated(arg2='foo', arg3='bar')
+
+    def test_region_failure(self):
+        secret = 'not json'
+        response = {}
+        versions = {
+            '01234567890123456789012345678901': ['AWSCURRENT']
+        }
+        version_response = {'SecretString': secret}
+        cache = SecretCache(client=self.get_client(response, versions, version_response, error=True))
+        caches = [SecretCache(client=self.get_client(response, versions, version_response, region_name='us-east-1'))]
+
+        @InjectSecretString('test', cache, caches)
+        def function_to_be_decorated(arg1, arg2, arg3):
+            self.assertEqual(arg1, secret)
+            self.assertEqual(arg2, 'foo')
+            self.assertEqual(arg3, 'bar')
+            return 'OK'
+
+        self.assertEqual(function_to_be_decorated('foo', 'bar'), 'OK')