Merge pull request #434 from qiniu/feat/middleware-and-backup-domains

add http client middleware and getting region hosts with backup domains

Author: bachue

qiniu/auth.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 import base64
+from datetime import datetime
 import hmac
+import os
 import time
 from hashlib import sha1
 from requests.auth import AuthBase
@@ -242,6 +244,14 @@ def __token(self, data):
         hashed = hmac.new(self.__secret_key, data, sha1)
         return urlsafe_base64_encode(hashed.digest())
 
+    @property
+    def should_sign_with_timestamp(self):
+        if self.disable_qiniu_timestamp_signature is not None:
+            return not self.disable_qiniu_timestamp_signature
+        if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE', '').lower() == 'true':
+            return False
+        return True
+
     def token_of_request(
             self,
             method,
@@ -294,12 +304,12 @@ def token_of_request(
         return '{0}:{1}'.format(self.__access_key, self.__token(data))
 
     def qiniu_headers(self, headers):
-        qiniu_fields = list(filter(
-            lambda k: k.startswith(self.qiniu_header_prefix) and len(k) > len(self.qiniu_header_prefix),
-            headers,
-        ))
-        return "\n".join([
-            "%s: %s" % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields)
+        qiniu_fields = [
+            key for key in headers
+            if key.startswith(self.qiniu_header_prefix) and len(key) > len(self.qiniu_header_prefix)
+        ]
+        return '\n'.join([
+            '%s: %s' % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields)
         ])
 
     @staticmethod
@@ -309,15 +319,30 @@ def __checkKey(access_key, secret_key):
 
 
 class QiniuMacRequestsAuth(AuthBase):
+    """
+    Attributes:
+        auth (QiniuMacAuth):
+    """
     def __init__(self, auth):
+        """
+        Args:
+            auth (QiniuMacAuth):
+        """
         self.auth = auth
 
     def __call__(self, r):
         if r.headers.get('Content-Type', None) is None:
             r.headers['Content-Type'] = 'application/x-www-form-urlencoded'
+
+        if self.auth.should_sign_with_timestamp:
+            x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+            r.headers['X-Qiniu-Date'] = x_qiniu_date
+
         token = self.auth.token_of_request(
-            r.method, r.headers.get('Host', None),
-            r.url, self.auth.qiniu_headers(r.headers),
+            r.method,
+            r.headers.get('Host', None),
+            r.url,
+            self.auth.qiniu_headers(r.headers),
             r.headers.get('Content-Type', None),
             r.body
         )

qiniu/config.py

@@ -1,20 +1,24 @@
 # -*- coding: utf-8 -*-
-
-from qiniu import zone
+from qiniu import region
 
 RS_HOST = 'http://rs.qiniu.com'  # 管理操作Host
 RSF_HOST = 'http://rsf.qbox.me'  # 列举操作Host
 API_HOST = 'http://api.qiniuapi.com'  # 数据处理操作Host
-UC_HOST = 'https://uc.qbox.me'  # 获取空间信息Host
+UC_HOST = region.UC_HOST  # 获取空间信息Host
 
 _BLOCK_SIZE = 1024 * 1024 * 4  # 断点续传分块大小，该参数为接口规格，暂不支持修改
 
 _config = {
-    'default_zone': zone.Zone(),
+    'default_zone': region.Region(),
     'default_rs_host': RS_HOST,
     'default_rsf_host': RSF_HOST,
     'default_api_host': API_HOST,
     'default_uc_host': UC_HOST,
+    'default_uc_backup_hosts': [
+        'kodo-config.qiniuapi.com',
+        'api.qiniu.com'
+    ],
+    'default_uc_backup_retry_times': 2,
     'connection_timeout': 30,  # 链接超时为时间为30s
     'connection_retries': 3,  # 链接重试次数为3次
     'connection_pool': 10,  # 链接池个数为10
@@ -27,6 +31,8 @@
     'default_rsf_host': False,
     'default_api_host': False,
     'default_uc_host': False,
+    'default_uc_backup_hosts': False,
+    'default_uc_backup_retry_times': False,
     'connection_timeout': False,
     'connection_retries': False,
     'connection_pool': False,
@@ -45,7 +51,8 @@ def get_default(key):
 def set_default(
         default_zone=None, connection_retries=None, connection_pool=None,
         connection_timeout=None, default_rs_host=None, default_uc_host=None,
-        default_rsf_host=None, default_api_host=None, default_upload_threshold=None):
+        default_rsf_host=None, default_api_host=None, default_upload_threshold=None,
+        default_uc_backup_hosts=None, default_uc_backup_retry_times=None):
     if default_zone:
         _config['default_zone'] = default_zone
         _is_customized_default['default_zone'] = True
@@ -61,6 +68,14 @@ def set_default(
     if default_uc_host:
         _config['default_uc_host'] = default_uc_host
         _is_customized_default['default_uc_host'] = True
+        _config['default_uc_backup_hosts'] = []
+        _is_customized_default['default_uc_backup_hosts'] = True
+    if default_uc_backup_hosts:
+        _config['default_uc_backup_hosts'] = default_uc_backup_hosts
+        _is_customized_default['default_uc_backup_hosts'] = True
+    if default_uc_backup_retry_times:
+        _config['default_uc_backup_retry_times'] = default_uc_backup_retry_times
+        _is_customized_default['default_uc_backup_retry_times'] = True
     if connection_retries:
         _config['connection_retries'] = connection_retries
         _is_customized_default['connection_retries'] = True

qiniu/http.py renamed to qiniu/http/__init__.py

@@ -1,16 +1,25 @@
 # -*- coding: utf-8 -*-
 import logging
-import os
 import platform
-from datetime import datetime
 
 import requests
+from requests.adapters import HTTPAdapter
 from requests.auth import AuthBase
 
-from qiniu.compat import is_py2, is_py3
-from qiniu import config
+from qiniu import config, __version__
 import qiniu.auth
-from . import __version__
+
+from .client import HTTPClient
+from .response import ResponseInfo
+from .middleware import UserAgentMiddleware
+
+
+qn_http_client = HTTPClient(
+    middlewares=[
+        UserAgentMiddleware(__version__)
+    ]
+)
+
 
 _sys_info = '{0}; {1}'.format(platform.system(), platform.machine())
 _python_ver = platform.python_version()
@@ -22,19 +31,6 @@
 _headers = {'User-Agent': USER_AGENT}
 
 
-def __add_auth_headers(headers, auth):
-    x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
-    if auth.disable_qiniu_timestamp_signature is not None:
-        if not auth.disable_qiniu_timestamp_signature:
-            headers['X-Qiniu-Date'] = x_qiniu_date
-    elif os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE'):
-        if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE').lower() != 'true':
-            headers['X-Qiniu-Date'] = x_qiniu_date
-    else:
-        headers['X-Qiniu-Date'] = x_qiniu_date
-    return headers
-
-
 def __return_wrapper(resp):
     if resp.status_code != 200 or resp.headers.get('X-Reqid') is None:
         return None, ResponseInfo(resp)
@@ -48,14 +44,15 @@ def __return_wrapper(resp):
 
 
 def _init():
-    session = requests.Session()
-    adapter = requests.adapters.HTTPAdapter(
+    global _session
+    if _session is None:
+        _session = qn_http_client.session
+
+    adapter = HTTPAdapter(
         pool_connections=config.get_default('connection_pool'),
         pool_maxsize=config.get_default('connection_pool'),
         max_retries=config.get_default('connection_retries'))
-    session.mount('http://', adapter)
-    global _session
-    _session = session
+    _session.mount('http://', adapter)
 
 
 def _post(url, data, files, auth, headers=None):
@@ -170,18 +167,16 @@ def _post_with_qiniu_mac(url, data, auth):
     qn_auth = qiniu.auth.QiniuMacRequestsAuth(
         auth
     ) if auth is not None else None
-    headers = __add_auth_headers({}, auth)
 
-    return _post(url, data, None, qn_auth, headers=headers)
+    return _post(url, data, None, qn_auth)
 
 
 def _get_with_qiniu_mac(url, params, auth):
     qn_auth = qiniu.auth.QiniuMacRequestsAuth(
         auth
     ) if auth is not None else None
-    headers = __add_auth_headers({}, auth)
 
-    return _get(url, params, qn_auth, headers=headers)
+    return _get(url, params, qn_auth)
 
 
 def _get_with_qiniu_mac_and_headers(url, params, auth, headers):
@@ -229,76 +224,3 @@ def _delete_with_qiniu_mac_and_headers(url, params, auth, headers):
     except Exception as e:
         return None, ResponseInfo(None, e)
     return __return_wrapper(r)
-
-
-class ResponseInfo(object):
-    """七牛HTTP请求返回信息类
-
-    该类主要是用于获取和解析对七牛发起各种请求后的响应包的header和body。
-
-    Attributes:
-        status_code: 整数变量，响应状态码
-        text_body:   字符串变量，响应的body
-        req_id:      字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers
-        x_log:       字符串变量，七牛HTTP扩展字段，参考 https://developer.qiniu.com/kodo/3924/common-request-headers
-        error:       字符串变量，响应的错误内容
-    """
-
-    def __init__(self, response, exception=None):
-        """用响应包和异常信息初始化ResponseInfo类"""
-        self.__response = response
-        self.exception = exception
-        if response is None:
-            self.status_code = -1
-            self.text_body = None
-            self.req_id = None
-            self.x_log = None
-            self.error = str(exception)
-        else:
-            self.status_code = response.status_code
-            self.text_body = response.text
-            self.req_id = response.headers.get('X-Reqid')
-            self.x_log = response.headers.get('X-Log')
-            if self.status_code >= 400:
-                if self.__check_json(response):
-                    ret = response.json() if response.text != '' else None
-                    if ret is None:
-                        self.error = 'unknown'
-                    else:
-                        self.error = response.text
-                else:
-                    self.error = response.text
-            if self.req_id is None and self.status_code == 200:
-                self.error = 'server is not qiniu'
-
-    def ok(self):
-        return self.status_code == 200 and self.req_id is not None
-
-    def need_retry(self):
-        if self.__response is None or self.req_id is None:
-            return True
-        code = self.status_code
-        if (code // 100 == 5 and code != 579) or code == 996:
-            return True
-        return False
-
-    def connect_failed(self):
-        return self.__response is None or self.req_id is None
-
-    def __str__(self):
-        if is_py2:
-            return ', '.join(
-                ['%s:%s' % item for item in self.__dict__.items()]).encode('utf-8')
-        elif is_py3:
-            return ', '.join(['%s:%s' %
-                              item for item in self.__dict__.items()])
-
-    def __repr__(self):
-        return self.__str__()
-
-    def __check_json(self, reponse):
-        try:
-            reponse.json()
-            return True
-        except Exception:
-            return False