Update code to retry web API calls for fetching datafile and pushing events

Author: Paul V Craven

optimizely/config_manager.py

@@ -19,6 +19,8 @@
 import threading
 from requests import codes as http_status_codes
 from requests import exceptions as requests_exceptions
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from . import exceptions as optimizely_exceptions
 from . import logger as optimizely_logger
@@ -200,6 +202,7 @@ def __init__(
         error_handler: Optional[BaseErrorHandler] = None,
         notification_center: Optional[NotificationCenter] = None,
         skip_json_validation: Optional[bool] = False,
+        retries: Optional[int] = 3,
     ):
         """ Initialize config manager. One of sdk_key or datafile has to be set to be able to use.
 
@@ -244,6 +247,7 @@ def __init__(
         self.stopped = threading.Event()
         self._initialize_thread()
         self._polling_thread.start()
+        self.retries = retries
 
     @staticmethod
     def get_datafile_url(sdk_key: Optional[str], url: Optional[str], url_template: Optional[str]) -> str:
@@ -391,9 +395,17 @@ def fetch_datafile(self) -> None:
             request_headers[enums.HTTPHeaders.IF_MODIFIED_SINCE] = self.last_modified
 
         try:
-            response = requests.get(
-                self.datafile_url, headers=request_headers, timeout=enums.ConfigManager.REQUEST_TIMEOUT,
-            )
+            print(f"Getting {self.datafile_url}")
+            session = requests.Session()
+
+            retries = Retry(total=self.retries,
+                            backoff_factor=0.1,
+                            status_forcelist=[ 500, 502, 503, 504 ])
+            adapter = HTTPAdapter(max_retries=retries)
+
+            session.mount('http://', adapter)
+            session.mount("https://", adapter)
+            response = session.get(self.datafile_url, timeout=enums.ConfigManager.REQUEST_TIMEOUT)            
         except requests_exceptions.RequestException as err:
             self.logger.error(f'Fetching datafile from {self.datafile_url} failed. Error: {err}')
             return
@@ -475,9 +487,17 @@ def fetch_datafile(self) -> None:
             request_headers[enums.HTTPHeaders.IF_MODIFIED_SINCE] = self.last_modified
 
         try:
-            response = requests.get(
-                self.datafile_url, headers=request_headers, timeout=enums.ConfigManager.REQUEST_TIMEOUT,
-            )
+            print(f"Getting {self.datafile_url}")
+            session = requests.Session()
+
+            retries = Retry(total=self.retries,
+                            backoff_factor=0.1,
+                            status_forcelist=[ 500, 502, 503, 504 ])
+            adapter = HTTPAdapter(max_retries=retries)
+
+            session.mount('http://', adapter)
+            session.mount("https://", adapter)
+            response = session.get(self.datafile_url, timeout=enums.ConfigManager.REQUEST_TIMEOUT) 
         except requests_exceptions.RequestException as err:
             self.logger.error(f'Fetching datafile from {self.datafile_url} failed. Error: {err}')
             return

optimizely/event_dispatcher.py

@@ -17,6 +17,8 @@
 
 import requests
 from requests import exceptions as request_exception
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 from . import event_builder
 from .helpers.enums import HTTPVerbs, EventDispatchConfig
@@ -44,11 +46,21 @@ def dispatch_event(event: event_builder.Event) -> None:
       event: Object holding information about the request to be dispatched to the Optimizely backend.
     """
         try:
+            session = requests.Session()
+
+            retries = Retry(total=EventDispatchConfig.REQUEST_TIMEOUT,
+                            backoff_factor=0.1,
+                            status_forcelist=[ 500, 502, 503, 504 ])
+            adapter = HTTPAdapter(max_retries=retries)
+
+            session.mount('http://', adapter)
+            session.mount("https://", adapter)
+
             if event.http_verb == HTTPVerbs.GET:
-                requests.get(event.url, params=event.params,
+                session.get(event.url, params=event.params,
                              timeout=EventDispatchConfig.REQUEST_TIMEOUT).raise_for_status()
             elif event.http_verb == HTTPVerbs.POST:
-                requests.post(
+                session.post(
                     event.url, data=json.dumps(event.params), headers=event.headers,
                     timeout=EventDispatchConfig.REQUEST_TIMEOUT,
                 ).raise_for_status()

optimizely/helpers/enums.py

@@ -198,6 +198,7 @@ class VersionType:
 class EventDispatchConfig:
     """Event dispatching configs."""
     REQUEST_TIMEOUT: Final = 10
+    RETRIES: Final = 2
 
 
 class OdpEventApiConfig: