Update KVS and its clients [WIP]

Author: vdusek

src/crawlee/_utils/file.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import contextlib
-import io
 import json
 import mimetypes
 import os
@@ -83,21 +82,6 @@ def determine_file_extension(content_type: str) -> str | None:
     return ext[1:] if ext is not None else ext
 
 
-def is_file_or_bytes(value: Any) -> bool:
-    """Determine if the input value is a file-like object or bytes.
-
-    This function checks whether the provided value is an instance of bytes, bytearray, or io.IOBase (file-like).
-    The method is simplified for common use cases and may not cover all edge cases.
-
-    Args:
-        value: The value to be checked.
-
-    Returns:
-        True if the value is either a file-like object or bytes, False otherwise.
-    """
-    return isinstance(value, (bytes, bytearray, io.IOBase))
-
-
 async def json_dumps(obj: Any) -> str:
     """Serialize an object to a JSON-formatted string with specific settings.
 

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -56,6 +56,7 @@ def item_count(self) -> int:
     @abstractmethod
     async def open(
         cls,
+        *,
         id: str | None,
         name: str | None,
         storage_dir: Path,
@@ -82,7 +83,7 @@ async def drop(self) -> None:
         """
 
     @abstractmethod
-    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, *, data: list[Any] | dict[str, Any]) -> None:
         """Push data to the dataset.
 
         The backend method for the `Dataset.push_data` call.

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -6,103 +6,139 @@
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
-    from contextlib import AbstractAsyncContextManager
+    from datetime import datetime
+    from pathlib import Path
 
-    from httpx import Response
+    from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreRecord
 
-    from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
+# Properties:
+# - id
+# - name
+# - created_at
+# - accessed_at
+# - modified_at
+
+# Methods:
+# - open
+# - drop
+# - get_value
+# - set_value
+# - delete_value
+# - iterate_keys
+# - get_public_url
 
 
 @docs_group('Abstract classes')
 class KeyValueStoreClient(ABC):
-    """An abstract class for key-value store resource clients.
+    """An abstract class for key-value store (KVS) resource clients.
 
     These clients are specific to the type of resource they manage and operate under a designated storage
     client, like a memory storage client.
     """
 
+    @property
     @abstractmethod
-    async def get(self) -> KeyValueStoreMetadata | None:
-        """Get metadata about the key-value store being managed by this client.
-
-        Returns:
-            An object containing the key-value store's details, or None if the key-value store does not exist.
-        """
+    def id(self) -> str:
+        """The ID of the key-value store."""
 
+    @property
     @abstractmethod
-    async def delete(self) -> None:
-        """Permanently delete the key-value store managed by this client."""
+    def name(self) -> str | None:
+        """The name of the key-value store."""
 
+    @property
     @abstractmethod
-    async def list_keys(
-        self,
-        *,
-        limit: int = 1000,
-        exclusive_start_key: str | None = None,
-    ) -> KeyValueStoreListKeysPage:
-        """List the keys in the key-value store.
+    def created_at(self) -> datetime:
+        """The time at which the key-value store was created."""
 
-        Args:
-            limit: Number of keys to be returned. Maximum value is 1000.
-            exclusive_start_key: All keys up to this one (including) are skipped from the result.
+    @property
+    @abstractmethod
+    def accessed_at(self) -> datetime:
+        """The time at which the key-value store was last accessed."""
 
-        Returns:
-            The list of keys in the key-value store matching the given arguments.
-        """
+    @property
+    @abstractmethod
+    def modified_at(self) -> datetime:
+        """The time at which the key-value store was last modified."""
 
+    @classmethod
     @abstractmethod
-    async def get_record(self, key: str) -> KeyValueStoreRecord | None:
-        """Retrieve the given record from the key-value store.
+    async def open(
+        cls,
+        *,
+        id: str | None,
+        name: str | None,
+        storage_dir: Path,
+    ) -> KeyValueStoreClient:
+        """Open existing or create a new key-value store client.
+
+        If a key-value store with the given name already exists, the appropriate key-value store client is returned.
+        Otherwise, a new key-value store is created and client for it is returned.
 
         Args:
-            key: Key of the record to retrieve.
+            id: The ID of the key-value store.
+            name: The name of the key-value store.
+            storage_dir: The path to the storage directory. If the client persists data, it should use this directory.
 
         Returns:
-            The requested record, or None, if the record does not exist
+            A key-value store client.
         """
 
     @abstractmethod
-    async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord[bytes] | None:
-        """Retrieve the given record from the key-value store, without parsing it.
+    async def drop(self) -> None:
+        """Drop the whole key-value store and remove all its items.
 
-        Args:
-            key: Key of the record to retrieve.
-
-        Returns:
-            The requested record, or None, if the record does not exist
+        The backend method for the `KeyValueStore.drop` call.
         """
 
     @abstractmethod
-    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
-        """Retrieve the given record from the key-value store, as a stream.
+    async def get_value(self, *, key: str) -> KeyValueStoreRecord | None:
+        """Retrieve the given record from the key-value store.
 
         Args:
             key: Key of the record to retrieve.
 
         Returns:
-            The requested record as a context-managed streaming Response, or None, if the record does not exist
+            The requested record, or None, if the record does not exist
         """
 
     @abstractmethod
-    async def set_record(self, key: str, value: Any, content_type: str | None = None) -> None:
+    async def set_value(self, *, key: str, value: Any, content_type: str | None = None) -> None:
         """Set a value to the given record in the key-value store.
 
         Args:
             key: The key of the record to save the value to.
             value: The value to save into the record.
-            content_type: The content type of the saved value.
+            content_type: The MIME content type string. If not provided, it is inferred from the value.
         """
 
     @abstractmethod
-    async def delete_record(self, key: str) -> None:
+    async def delete_value(self, *, key: str) -> None:
         """Delete the specified record from the key-value store.
 
         Args:
             key: The key of the record which to delete.
         """
 
     @abstractmethod
-    async def get_public_url(self, key: str) -> str:
+    async def iterate_keys(
+        self,
+        *,
+        exclusive_start_key: str | None = None,
+        limit: int = 1000,
+    ) -> KeyValueStoreListKeysPage:
+        """List the keys in the key-value store.
+
+        Args:
+            exclusive_start_key: All keys up to this one (including) are skipped from the result.
+            limit: Number of keys to be returned.
+
+        Returns:
+            The list of keys in the key-value store matching the given arguments.
+        """
+
+    @abstractmethod
+    async def get_public_url(self, *, key: str) -> str:
         """Get the public URL for the given key.
 
         Args:

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -10,12 +10,11 @@
 from pydantic import ValidationError
 from typing_extensions import override
 
-from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
-from ._utils import json_dumps
+from ._utils import METADATA_FILENAME, json_dumps
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator