add .qiniu_pythonsdk_hostscache.json file ptah parameter and update fetch returns info

Author: yangjunren

qiniu/region.py

@@ -22,37 +22,42 @@ def __init__(
             up_host_backup=None,
             io_host=None,
             host_cache={},
-            scheme="http",
-            home_dir=os.getcwd()):
+            home_dir=None,
+            scheme="http"):
         """初始化Zone类"""
-        self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host
+        self.up_host, self.up_host_backup, self.io_host, self.home_dir = up_host, up_host_backup, io_host, home_dir
         self.host_cache = host_cache
         self.scheme = scheme
-        self.home_dir = home_dir
 
-    def get_up_host_by_token(self, up_token):
+    def get_up_host_by_token(self, up_token, home_dir):
         ak, bucket = self.unmarshal_up_token(up_token)
-        up_hosts = self.get_up_host(ak, bucket)
+        if home_dir is None:
+            home_dir = os.getcwd()
+        up_hosts = self.get_up_host(ak, bucket, home_dir)
         return up_hosts[0]
 
-    def get_up_host_backup_by_token(self, up_token):
+    def get_up_host_backup_by_token(self, up_token, home_dir):
         ak, bucket = self.unmarshal_up_token(up_token)
-        up_hosts = self.get_up_host(ak, bucket)
+        if home_dir is None:
+            home_dir = os.getcwd()
+        up_hosts = self.get_up_host(ak, bucket, home_dir)
         if (len(up_hosts) <= 1):
             up_host = up_hosts[0]
         else:
             up_host = up_hosts[1]
         return up_host
 
-    def get_io_host(self, ak, bucket):
+    def get_io_host(self, ak, bucket, home_dir):
         if self.io_host:
             return self.io_host
-        bucket_hosts = self.get_bucket_hosts(ak, bucket)
+        if home_dir is None:
+            home_dir = os.getcwd()
+        bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir)
         io_hosts = bucket_hosts['ioHosts']
         return io_hosts[0]
 
-    def get_up_host(self, ak, bucket):
-        bucket_hosts = self.get_bucket_hosts(ak, bucket)
+    def get_up_host(self, ak, bucket, home_dir):
+        bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir)
         up_hosts = bucket_hosts['upHosts']
         return up_hosts
 
@@ -74,10 +79,10 @@ def unmarshal_up_token(self, up_token):
 
         return ak, bucket
 
-    def get_bucket_hosts(self, ak, bucket):
+    def get_bucket_hosts(self, ak, bucket, home_dir):
         key = self.scheme + ":" + ak + ":" + bucket
 
-        bucket_hosts = self.get_bucket_hosts_to_cache(key)
+        bucket_hosts = self.get_bucket_hosts_to_cache(key, home_dir)
         if (len(bucket_hosts) > 0):
             return bucket_hosts
 
@@ -113,14 +118,14 @@ def get_bucket_hosts(self, ak, bucket):
             'ioHosts': scheme_hosts['io'],
             'deadline': int(time.time()) + hosts['ttl']
         }
-
-        self.set_bucket_hosts_to_cache(key, bucket_hosts)
+        home_dir = ""
+        self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir)
         return bucket_hosts
 
-    def get_bucket_hosts_to_cache(self, key):
+    def get_bucket_hosts_to_cache(self, key, home_dir):
         ret = []
         if (len(self.host_cache) == 0):
-            self.host_cache_from_file()
+            self.host_cache_from_file(home_dir)
 
         if (not (key in self.host_cache)):
             return ret
@@ -130,12 +135,14 @@ def get_bucket_hosts_to_cache(self, key):
 
         return ret
 
-    def set_bucket_hosts_to_cache(self, key, val):
+    def set_bucket_hosts_to_cache(self, key, val, home_dir):
         self.host_cache[key] = val
-        self.host_cache_to_file()
+        self.host_cache_to_file(home_dir)
         return
 
-    def host_cache_from_file(self):
+    def host_cache_from_file(self, home_dir):
+        if home_dir is not None:
+            self.home_dir = home_dir
         path = self.host_cache_file_path()
         if not os.path.isfile(path):
             return None
@@ -148,7 +155,7 @@ def host_cache_from_file(self):
     def host_cache_file_path(self):
         return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json")
 
-    def host_cache_to_file(self):
+    def host_cache_to_file(self, home_dir):
         path = self.host_cache_file_path()
         with open(path, 'w') as f:
             compat.json.dump(self.host_cache, f)

qiniu/services/storage/bucket.py

@@ -161,25 +161,28 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'):
         to = entry(bucket_to, key_to)
         return self.__rs_do('copy', resource, to, 'force/{0}'.format(force))
 
-    def fetch(self, url, bucket, key=None):
+    def fetch(self, url, bucket, key=None, hostscache_dir=None):
         """抓取文件:
         从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考：
         http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html
 
         Args:
-            url:    指定的URL
-            bucket: 目标资源空间
-            key:    目标资源文件名
+            url:      指定的URL
+            bucket:   目标资源空间
+            key:      目标资源文件名
+            hostscache_dir： host请求 缓存文件保存位置
 
         Returns:
-            一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"}
+            一个dict变量：
+                成功 返回{'fsize': <fsize int>, 'hash': <hash string>, 'key': <key string>, 'mimeType': <mimeType string>}
+                失败 返回 None
             一个ResponseInfo对象
         """
         resource = urlsafe_base64_encode(url)
         to = entry(bucket, key)
-        return self.__io_do(bucket, 'fetch', resource, 'to/{0}'.format(to))
+        return self.__io_do(bucket, 'fetch', hostscache_dir, resource, 'to/{0}'.format(to))
 
-    def prefetch(self, bucket, key):
+    def prefetch(self, bucket, key, hostscache_dir=None):
         """镜像回源预取文件:
 
         从镜像源站抓取资源到空间中，如果空间中已经存在，则覆盖该资源，具体规格参考
@@ -188,13 +191,14 @@ def prefetch(self, bucket, key):
         Args:
             bucket: 待获取资源所在的空间
             key:    代获取资源文件名
+            hostscache_dir： host请求 缓存文件保存位置
 
         Returns:
             一个dict变量，成功返回NULL，失败返回{"error": "<errMsg string>"}
             一个ResponseInfo对象
         """
         resource = entry(bucket, key)
-        return self.__io_do(bucket, 'prefetch', resource)
+        return self.__io_do(bucket, 'prefetch', hostscache_dir, resource)
 
     def change_mime(self, bucket, key, mime):
         """修改文件mimeType:
@@ -349,9 +353,9 @@ def __uc_do(self, operation, *args):
     def __rs_do(self, operation, *args):
         return self.__server_do(config.get_default('default_rs_host'), operation, *args)
 
-    def __io_do(self, bucket, operation, *args):
+    def __io_do(self, bucket, operation, home_dir, *args):
         ak = self.auth.get_access_key()
-        io_host = self.zone.get_io_host(ak, bucket)
+        io_host = self.zone.get_io_host(ak, bucket, home_dir)
         return self.__server_do(io_host, operation, *args)
 
     def __server_do(self, host, operation, *args):

qiniu/services/storage/uploader.py

@@ -11,7 +11,7 @@
 
 def put_data(
         up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None,
-        fname=None):
+        fname=None, hostscache_dir=None):
     """上传二进制流到七牛
 
     Args:
@@ -22,6 +22,7 @@ def put_data(
         mime_type:        上传数据的mimeType
         check_crc:        是否校验crc32
         progress_handler: 上传进度
+        hostscache_dir：  host请求 缓存文件保存位置
 
     Returns:
         一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
@@ -39,23 +40,24 @@ def put_data(
         final_data = data
 
     crc = crc32(final_data)
-    return _form_put(up_token, key, final_data, params, mime_type, crc, progress_handler, fname)
+    return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname)
 
 
 def put_file(up_token, key, file_path, params=None,
              mime_type='application/octet-stream', check_crc=False,
-             progress_handler=None, upload_progress_recorder=None, keep_last_modified=False):
+             progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None):
     """上传文件到七牛
 
     Args:
-        up_token:         上传凭证
-        key:              上传文件名
-        file_path:        上传文件的路径
-        params:           自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
-        mime_type:        上传数据的mimeType
-        check_crc:        是否校验crc32
-        progress_handler: 上传进度
+        up_token:                 上传凭证
+        key:                      上传文件名
+        file_path:                上传文件的路径
+        params:                   自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
+        mime_type:                上传数据的mimeType
+        check_crc:                是否校验crc32
+        progress_handler:         上传进度
         upload_progress_recorder: 记录上传进度，用于断点续传
+        hostscache_dir：          host请求 缓存文件保存位置
 
     Returns:
         一个dict变量，类似 {"hash": "<Hash string>", "key": "<Key string>"}
@@ -68,19 +70,20 @@ def put_file(up_token, key, file_path, params=None,
         file_name = os.path.basename(file_path)
         modify_time = int(os.path.getmtime(file_path))
         if size > config._BLOCK_SIZE * 2:
-            ret, info = put_stream(up_token, key, input_stream, file_name, size, params,
+            ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params,
                                    mime_type, progress_handler,
                                    upload_progress_recorder=upload_progress_recorder,
                                    modify_time=modify_time, keep_last_modified=keep_last_modified)
         else:
             crc = file_crc32(file_path)
             ret, info = _form_put(up_token, key, input_stream, params, mime_type,
-                                  crc, progress_handler, file_name,
+                                  crc, hostscache_dir, progress_handler, file_name,
                                   modify_time=modify_time, keep_last_modified=keep_last_modified)
     return ret, info
 
 
-def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None, modify_time=None,
+def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None,
+              modify_time=None,
               keep_last_modified=False):
     fields = {}
     if params:
@@ -95,7 +98,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None
     if config.get_default('default_zone').up_host:
         url = config.get_default('default_zone').up_host
     else:
-        url = config.get_default('default_zone').get_up_host_by_token(up_token)
+        url = config.get_default('default_zone').get_up_host_by_token(up_token, hostscache_dir)
     # name = key if key else file_name
 
     fname = file_name
@@ -112,7 +115,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None
             if config.get_default('default_zone').up_host_backup:
                 url = config.get_default('default_zone').up_host_backup
             else:
-                url = config.get_default('default_zone').get_up_host_backup_by_token(up_token)
+                url = config.get_default('default_zone').get_up_host_backup_by_token(up_token, hostscache_dir)
         if hasattr(data, 'read') is False:
             pass
         elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()):
@@ -124,11 +127,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None
     return r, info
 
 
-def put_stream(up_token, key, input_stream, file_name, data_size, params=None,
+def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None,
                mime_type=None, progress_handler=None,
                upload_progress_recorder=None, modify_time=None, keep_last_modified=False):
-    task = _Resume(up_token, key, input_stream, data_size, params, mime_type,
-                   progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified)
+    task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type,
+                   progress_handler, upload_progress_recorder, modify_time, keep_last_modified)
     return task.upload()
 
 
@@ -140,30 +143,32 @@ class _Resume(object):
     http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html
 
     Attributes:
-        up_token:         上传凭证
-        key:              上传文件名
-        input_stream:     上传二进制流
-        data_size:        上传流大小
-        params:           自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
-        mime_type:        上传数据的mimeType
-        progress_handler: 上传进度
-        upload_progress_recorder:  记录上传进度，用于断点续传
-        modify_time:      上传文件修改日期
+        up_token:                   上传凭证
+        key:                        上传文件名
+        input_stream:               上传二进制流
+        data_size:                  上传流大小
+        params:                     自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar
+        mime_type:                  上传数据的mimeType
+        progress_handler:           上传进度
+        upload_progress_recorder:   记录上传进度，用于断点续传
+        modify_time:                上传文件修改日期
+        hostscache_dir：            host请求 缓存文件保存位置
     """
 
-    def __init__(self, up_token, key, input_stream, data_size, params, mime_type,
-                 progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified):
+    def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type,
+                 progress_handler, upload_progress_recorder, modify_time, keep_last_modified):
         """初始化断点续上传"""
         self.up_token = up_token
         self.key = key
         self.input_stream = input_stream
+        self.file_name = file_name
         self.size = data_size
+        self.hostscache_dir = hostscache_dir
         self.params = params
         self.mime_type = mime_type
         self.progress_handler = progress_handler
         self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder()
         self.modify_time = modify_time or time.time()
-        self.file_name = file_name
         self.keep_last_modified = keep_last_modified
         # print(self.modify_time)
         # print(modify_time)
@@ -186,7 +191,7 @@ def recovery_from_record(self):
 
         try:
             if not record['modify_time'] or record['size'] != self.size or \
-                            record['modify_time'] != self.modify_time:
+                    record['modify_time'] != self.modify_time:
                 return 0
         except KeyError:
             return 0
@@ -199,7 +204,7 @@ def upload(self):
         if config.get_default('default_zone').up_host:
             host = config.get_default('default_zone').up_host
         else:
-            host = config.get_default('default_zone').get_up_host_by_token(self.up_token)
+            host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir)
         offset = self.recovery_from_record()
         for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset):
             length = len(block)
@@ -211,7 +216,8 @@ def upload(self):
                 if config.get_default('default_zone').up_host_backup:
                     host = config.get_default('default_zone').up_host_backup
                 else:
-                    host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token)
+                    host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token,
+                                                                                          self.hostscache_dir)
             if info.need_retry() or crc != ret['crc32']:
                 ret, info = self.make_block(block, length, host)
                 if ret is None or crc != ret['crc32']: