diff --git a/_build_scripts/slack-find-author.sh b/_build_scripts/slack-find-author.sh index b910d619..af037601 100644 --- a/_build_scripts/slack-find-author.sh +++ b/_build_scripts/slack-find-author.sh @@ -10,6 +10,7 @@ git_slack_map=( ["Charlie Harr"]="<@U044XTHRVFA>" ["Connor Shorten"]="<@U03FRH53SUT>" ["Daniel Madalitso Phiri"]="<@U060UJ41YBC>" + ["DanielleWashington"]="<@U088SBVDCET>" ["Dirk Kulawiak"]="<@U03MWHK4KV3>" ["Duda Nogueira"]="<@U05K3K9M82F>" ["dyma solovei"]="<@U07NGR323JR>" diff --git a/docs/deploy/aws/_category_.json b/docs/deploy/aws/_category_.json deleted file mode 100644 index 13cc9bd6..00000000 --- a/docs/deploy/aws/_category_.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "label": "AWS", - "position": 1 -} \ No newline at end of file diff --git a/docs/deploy/aws/index.mdx b/docs/deploy/aws/index.mdx deleted file mode 100644 index 713a3823..00000000 --- a/docs/deploy/aws/index.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: AWS and Weaviate -description: AWS docs -sidebar_position: 0 ---- - -# Tacetve a gemit quisquam praestat meruisse gradientis - -## Nec sum vagatur nupta tantus - -Lorem markdownum pugnando ventoque et ducta et et adurat locus inde cervos -refeci Phaethon clivum linguaque Salmacis geras coeperat mollia. Leucothoen Ceyx -est viret nostraque; meum utque latens -quae, patrios plebe inpositum: torvum. **Stipite fluminaque videri**; ictu -duabus sua *condita monstrum*. Imo undis: aut niveo sidoniae; placato sorores -at, quae perque. *Candore dolore*, et secum rastra ceratis et sollicitat vincula -matri mei alios in rigore murmur foedataque, cum. - -Et solvit ex comitante super luctus. Unda posse perque venitque quod algae -lymphis postquam optime. - -## Flammis levi harena illi membra caudam - -Ipse inducitur stridore Aiacem intonsum! Est arma, hausitque *tu* litora -aetasque, at fuit Aegea vides. Non laborum longi fortisque *putes bracchia* -umbras te fingit enim manum! - -Ingentia sive Idaeo serpere, dives inpia. Oculos in aras ergo bracchiaque vera, -omnis solutus vellet: et. Est et putat nec obvia unda -totidem amisit *coniurataeque sic arabat* mea. Caune -quodque **addita** truncis quid latebra: incubat pallorque turribus et ignarus -signum pectore quicquid Abantiades locus. **Ante** aliqua Aiacis deferre, ferunt -hunc omnia, et hic Aconteus nec. - -Odit festas dixit et quas fidissima, *est umbrae matrona* palude. Ait et quod -pectora circa. - -## Elegit sim ulla poteramus stridente cessit auras - -Tanta *Elinque*. Mea quae cultum Palameden Aglauros si sacra miserere quam ego, -**e scires** imo sonitusque tumidarum fremit quae ictu? Tum est sub vate vasto, -Galanthida in potui huc et temptat Cenaeo praeterque septem caede. - -> Suo ipse cubitique et vento in onerosa adunco, non. **In paelice Quirites** -> sunt stant sinebat superstes differt non per potentia illiusque nostris -> Hecateidos? Tempore haud imagine facta ora, uni quam fatorum. - -Altis animalibus pampinus venit mirabilis busto: tamen hunc, **salutant** -Stheneleia vultibus pectore. Evocat posuere anxia. - -Aper est; in tuum medius, responderat tamen: magis vix! Amor regno tamen tuli -remige facitote **reverentia**: loqui vade pignora. diff --git a/docs/deploy/config-guides/async-rep.md b/docs/deploy/config-guides/async-rep.md new file mode 100644 index 00000000..3bed2b78 --- /dev/null +++ b/docs/deploy/config-guides/async-rep.md @@ -0,0 +1,179 @@ +--- + +title: Async Replication + +--- + +Introduced to GA in the 1.29 release, Async Replication is a mechanism used to ensure eventual consistency across nodes in a distributed cluster. It works as a background process that automatically keeps nodes in sync without requiring user queries. Previously, consistency was achieved through "read repair" which involved nodes comparing data during a read request and exchanging missing or outdated information. This approach guarantees eventual consistency without requiring read operations. + +:::info + +This applies solely to data objects, as metadata consistency is treated differently (through RAFT consensus). +::: + +### Under the Hood + +- Async replication operates as a background process either per tenant (in a multi-tenant collection) or per shard (in a non-multi-tenant collection). +- It is disabled by default but can be enabled through collection configuration changes, similar to setting the replication factor. + +## Environment Variable Deep Dive + +These environment variables can be used to fine-tune behavior for your specific use case or deployment environment. + +:::tip +The optimal values for these variables will ultimately depend on factors like: data size, network conditions, write patterns, and the desired level of eventual consistency. +::: + +## Use Cases + +### General + +
+ + Feature Control +#### `ASYNC_REPLICATION_DISABLED` +Globally disables the entire async replication feature. + +- Its default value is `false`. +- **Use case**: This is useful when you have many tenants or collections where a temporary global disable is needed, like during debugging or critical maintenance. +- **Special Considerations**: + - This overrides any collection configuration. + +
+ +
+Replication Control + +#### `ASYNC_REPLICATION_PROPAGATION_LIMIT` +Defines the maximum number of objects that will be propagated in a single async replication iteration (after one hash tree comparison). + - By default is set to 10,000. + - **Use Case(s)**: Can be adjusted based on network capacity and the desired rate of convergence. + - **Considerations**: Even if more than this number of differences are detected, only this many objects will be propagated in the current iteration. Subsequent iterations will handle the remaining differences. + + +#### `ASYNC_REPLICATION_PROPAGATION_DELAY` +Introduces a delay before considering an object for propagation. Only objects older than this delay are considered. + - By default it is set to 30 seconds. + - **Use Case(s)**: If an object is inserted into one node but the insertion is still in progress, the hash comparison might detect it. This delay prevents the async replication from trying to propagate it before the local write operation is fully complete. + - **Considerations**: This should be set based on the typical write latency of the system. +
+ +
+ Operational Visibility + +#### `ASYNC_REPLICATION_LOGGING_FREQUENCY` +Controls how often the background async replication process logs its activity. + - By default it is set to 5 seconds. + - **Use Case(s)**: Increasing the frequency provides more detailed logs, while decreasing it reduces log verbosity. +
+ +### Performance Tuning + +
+ + Memory Optimization + +#### `ASYNC_REPLICATION_HASHTREE_HEIGHT` +Customizes the height of the hash tree built by each node to represent its locally stored data. +- By default the value is set to 16 which is roughly 2MB of RAM per shard on each node. +- **Use case(s)**: + - In multi-tenant setups with a large number of tenants, reducing the hash tree would minimize the memory footprint. + - For very large collections, a larger hash tree could be more beneficial for more efficient identification of differing data ranges. +- **Special Considerations**: + - Modification of the hash tree height requires rebuilding the hash tree on each node, which involves iterating over all existing objects. + +
+ +
+ + Throughput and Concurrency + +#### `ASYNC_REPLICATION_PROPAGATION_CONCURRENCY` +Controls the number of concurrent goroutines (or threads) used to send batches of objects during the propagation phase. + - By default it is set to 5. + - **Considerations**: Increasing concurrency can improve propagation speed, but needs to be balanced with potential resource contention (CPU, network). + +
+ +
+ + Batch Processing + +#### `ASYNC_REPLICATION_DIFF_BATCH_SIZE` +Sets the number of object metadata fetched per request during the comparison phase. + - By default it is set to 1000. + - **Use Case(s)**: May be increased to potentially improve performance if network latency is low and nodes can handle larger requests. + - **Considerations**: Fetching metadata in batches optimizes network communication. + + +#### `ASYNC_REPLICATION_PROPAGATION_BATCH_SIZE` +Sets the maximum number of objects included in each batch when propagating data to a remote node. + - By default is set to 100. + - **Use Case(s)**: + - For large objects, reducing the batch size can help manage memory usage during propagation. The batch size could be similar to the batch size used during initial data insertion. + - For smaller objects, increasing the batch size might improve propagation efficiency by reducing the overhead of individual requests, but needs to be balanced with potential memory pressure. + - **Considerations**: This setting is particularly important for large objects, as larger batches can lead to higher memory consumption during transmission. Multiple batches may be sent within a single iteration to reach the `ASYNC_REPLICATION_PROPAGATION_LIMIT`. + +
+ +### Consistency Tuning + +
+ + Synchronization Frequency +#### `ASYNC_REPLICATION_FREQUENCY` +Defines how often each node initiates the process of comparing its local data (via the hash tree) with other nodes storing the same shard. This regularly checks for inconsistencies, even if no changes have been explicitly triggered. +- It's default value is 30 seconds. +- **Use Case(s)** + - Decreasing the frequency can be beneficial for applications that require faster convergence to eventual consistency. + - Increasing the frequency can be beneficial for reducing the load on the system by relaxing the eventual consistency. + +#### `ASYNC_REPLICATION_FREQUENCY_WHILE_PROPAGATING` +Defines a shorter frequency for subsequent comparison and propagation attempts when a previous propagation cycle did not complete (i.e., not all detected differences were synchronized). + - By default it is set to 20 milliseconds. + - **Use Case(s)**: When inconsistencies are known to exist, this expedites the synchronization process. + - **Considerations**: This is activated after a propagation cycle detects differences but does not propagate all of them due to limits. + +
+ +
+ Node Status Monitoring + +#### `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` +Defines the frequency at which the system checks for changes in the availability of nodes within the cluster. + - By default it is set to 5 seconds. + - **Use Case(s)**: When a node rejoins the cluster after a period of downtime, it is highly likely to be out of sync. This setting ensures that the replication process is initiated promptly. + +
+ +
+Timeout Management + +#### `ASYNC_REPLICATION_DIFF_PER_NODE_TIMEOUT` +Defines the maximum time to wait for a response when requesting object metadata from a remote node during the comparison phase, this prevents indefinite blocking if a node is unresponsive. + - By default is set to 10 seconds. + - **Use Case(s)**: May need to be increased in environments with high network latency or potentially slow-responding nodes. + +#### `ASYNC_REPLICATION_PROPAGATION_TIMEOUT` +Sets the maximum time allowed for a single propagation request (sending actual object data) to a remote node. + - By default is set to 30 seconds. + - **Use Case(s)**: May need to be increased in scenarios with high network latency, large object sizes (e.g., images, vectors), or when sending large batches of objects. + - **Considerations**: Network latency, batch size, and the size of the objects being propagated can all affect timeouts. + +
+ + + +### Further Resources + +[Concepts: Replication](https://weaviate.io/developers/weaviate/concepts/replication-architecture/consistency) + +[Replication How-To](https://weaviate.io/developers/weaviate/configuration/replication#async-replication-settings) + +[Environment Variables](https://weaviate.io/developers/weaviate/config-refs/env-vars#async-replication) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/config-guides/authentication.md b/docs/deploy/config-guides/authentication.md new file mode 100644 index 00000000..783a1f5f --- /dev/null +++ b/docs/deploy/config-guides/authentication.md @@ -0,0 +1,338 @@ +--- +title: Authentication +image: og/docs/configuration.jpg +# tags: ['authentication'] +--- + +:::info Authentication and authorization +Authentication and authorization are closely related concepts, and sometimes abbreviated as `AuthN` and `AuthZ`. Authentication (`AuthN`) is the process of verifying the identity of a user, while authorization (`AuthZ`) is the process of determining what permissions the user has. +::: + +Weaviate controls access through user authentication via API keys or OpenID Connect (OIDC), with an option for anonymous access. Users can then be assigned different [authorization](./authorization.md) levels, as shown in the diagram below. + +```mermaid +flowchart LR + %% Define main nodes + Request["Client
Request"] + AuthCheck{"AuthN
Enabled?"} + AccessCheck{"Check
AuthZ"} + Access["✅ Access
Granted"] + Denied["❌ Access
Denied"] + + %% Define authentication method nodes + subgraph auth ["AuthN"] + direction LR + API["API Key"] + OIDC["OIDC"] + AuthResult{"Success?"} + end + + %% Define connections + Request --> AuthCheck + AuthCheck -->|"No"| AccessCheck + AuthCheck -->|"Yes"| auth + API --> AuthResult + OIDC --> AuthResult + AuthResult -->|"Yes"| AccessCheck + AuthResult -->|"No"| Denied + + AccessCheck -->|"Pass"| Access + AccessCheck -->|"Fail"| Denied + + %% Style nodes + style Request fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style AuthCheck fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style AccessCheck fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Access fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style Denied fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style API fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style OIDC fill:#ffffff,stroke:#B9C8DF,color:#130C49 + style AuthResult fill:#ffffff,stroke:#B9C8DF,color:#130C49 + + %% Style subgraph + style auth fill:#ffffff,stroke:#130C49,stroke-width:2px,color:#130C49 +``` + +For example, a user logging in with the API key `jane-secret` may be granted administrator permissions, while another user logging in with the API key `ian-secret` may be granted read-only permissions. + +In summary, Weaviate allows the following authentication methods: + +- API key +- OpenID Connect (OIDC) +- Anonymous access **(no authentication, strongly discouraged except for development or evaluation)** + +Note that API key and OIDC authentication can be both enabled at the same time. + +The way to configure authentication differs by your deployment method, depending on whether you are running Weaviate in Docker or Kubernetes. Below, we provide examples for both. + +:::info What about Weaviate Cloud (WCD)? +For Weaviate Cloud (WCD) instances, authentication is pre-configured with OIDC and API key access. You can [authenticate against Weaviate](../connections/connect-cloud.mdx) with your WCD credentials using OIDC, or [with API keys](/docs/cloud/platform/manage-api-keys). +::: + +## API Key Authentication + +API key authentication is a simple and effective way to authenticate users. Each user is assigned a unique API key, which is used to authenticate the user. + +Note that you can either: +- Set one user for all API keys, or +- Define one user per API key (the number of users must match the number of API keys) + +Make sure all listed users are also configured in the authorization settings. + +### API keys: Docker + +API key authentication can be configured using environment variables. In Docker Compose, set them in the configuration file (`docker-compose.yml`) such as in the following example: + +```yaml +services: + weaviate: + ... + environment: + ... + # Disable anonymous access. + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false' + + # Enables API key authentication. + AUTHENTICATION_APIKEY_ENABLED: 'true' + + # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below. + AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'user-a-key,user-b-key' + + # List one or more user identities, separated by commas. Each identity corresponds to a specific key above. + AUTHENTICATION_APIKEY_USERS: 'user-a,user-b' +``` + +This configuration: +- Disables anonymous access +- Enables API key authentication +- Defines plaintext API keys in `AUTHENTICATION_APIKEY_ALLOWED_KEYS` +- Associates users with the API keys in `AUTHENTICATION_APIKEY_USERS` + +These users can now be assigned permissions based on the authorization settings. + +### API keys: Kubernetes + +For Kubernetes deployments using Helm, API key authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration: + +```yaml +authentication: + anonymous_access: + # Disable anonymous access. + enabled: false + + apikey: + # Enables API key authentication. + enabled: true + + # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below. + allowed_keys: + - user-a-key + - user-b-key + + # List one or more user identities, separated by commas. Each identity corresponds to a specific key above. + users: + - user-a + - user-b +``` + +This configuration: + +- Disables anonymous access +- Enables API key authentication +- Defines plaintext API keys in `allowed_keys` +- Associates users with the API keys in `users` + +:::warning Environment Variables Take Precedence +If you configure API keys using environment variables, those settings will take precedence over the values in `values.yaml`. To use the Helm values configuration, ensure you have not set the corresponding environment variables. +::: + +For enhanced security in production environments, you can store API keys in Kubernetes secrets and reference them using environment variables instead of storing them as plaintext in the Helm values. + +## OIDC Authentication + +OIDC authentication requires the resource (Weaviate) to validate tokens issued by an identity provider. The identity provider authenticates the user and issues tokens, which are then validated by Weaviate. + +In an example setup, a Weaviate instance acts as the resource, Weaviate Cloud (WCD) acts as the identity provider, and the Weaviate client acts on behalf of the user. + +Any "OpenID Connect" compatible token issuer that implements OpenID Connect Discovery is compatible with Weaviate. + +This document discusses how to configure Weaviate as the resource. + +
+ + More about OIDC + + +With [OpenID Connect](https://openid.net/connect/) (based on OAuth2), an +external identity provider and token issuer ('token issuer' hereafter) is responsible for managing users. + +OIDC authentication requires obtaining a valid token from the token issuer so that it can be sent in the header of any request to Weaviate. This applies to both REST and GraphQL requests. + +When Weaviate receives a token (JSON Web Token or JWT), it verifies +that it was indeed signed by the configured token issuer. If the signature is +correct, all contents of the token are trusted, which authenticates the user based on the information in the token. + +
+ +### OIDC: Docker + +To configure Weaviate for OIDC-based authentication, add the following environment variables to your configuration file. + +An example `docker-compose.yml` file looks like this: + +```yaml +services: + weaviate: + ... + environment: + ... + # enabled (optional - defaults to false) turns OIDC auth on. All other fields in + # this section will only be validated if enabled is set to true. + AUTHENTICATION_OIDC_ENABLED: 'true' + + # issuer (required) tells weaviate how to discover the token issuer. This + # endpoint must implement the OpenID Connect Discovery spec, so that weaviate + # can retrieve the issuer's public key. + # + # The example URL below uses the path structure commonly found with keycloak + # where an example realm 'my-weaviate-usecase' was created. The exact + # path structure depends on the token issuer. See the token issuer's documentation + # about which endpoint implements OIDC Discovery. + AUTHENTICATION_OIDC_ISSUER: 'http://my-token-issuer/auth/realms/my-weaviate-usecase' + + # client_id (required unless skip_client_id_check is set to true) tells + # Weaviate to check for a particular OAuth 2.0 client_id in the audience claim. + # This is to prevent that a token which was signed by the correct issuer + # but never intended to be used with Weaviate can be used for authentication. + # + # For more information on what clients are in OAuth 2.0, see + # https://tools.ietf.org/html/rfc6749#section-1.1 + AUTHENTICATION_OIDC_CLIENT_ID: 'my-weaviate-client' + + # username_claim (required) tells Weaviate which claim in the token to use for extracting + # the username. The username will be passed to the authorization plugin. + AUTHENTICATION_OIDC_USERNAME_CLAIM: 'email' + + # skip_client_id_check (optional, defaults to false) skips the client_id + # validation in the audience claim as outlined in the section above. + # Not recommended to set this option as it reduces security, only set this + # if your token issuer is unable to provide a correct audience claim + AUTHENTICATION_OIDC_SKIP_CLIENT_ID_CHECK: 'false' + + # scope (optional) these will be used by clients as default scopes for authentication + AUTHENTICATION_OIDC_SCOPES: '' +``` + +:::info OIDC and Azure +As of November 2022, we were aware of some differences in Microsoft Azure's OIDC implementation compared to others. If you are using Azure and experiencing difficulties, [this external blog post](https://xsreality.medium.com/making-azure-ad-oidc-compliant-5734b70c43ff) may be useful. +::: + +### OIDC: Kubernetes + +For Kubernetes deployments using Helm, OIDC authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration: + +```yaml +authentication: + anonymous_access: + # Disable anonymous access. + enabled: false + oidc: + # enabled (optional - defaults to false) turns OIDC auth on. All other fields in + # this section will only be validated if enabled is set to true. + enabled: true + + # issuer (required) tells weaviate how to discover the token issuer. This + # endpoint must implement the OpenID Connect Discovery spec, so that weaviate + # can retrieve the issuer's public key. + # + # The example URL below uses the path structure commonly found with keycloak + # where an example realm 'my-weaviate-usecase' was created. The exact + # path structure depends on the token issuer. See the token issuer's documentation + # about which endpoint implements OIDC Discovery. + issuer: 'http://my-token-issuer/auth/realms/my-weaviate-usecase' + + # client_id (required unless skip_client_id_check is set to true) tells + # Weaviate to check for a particular OAuth 2.0 client_id in the audience claim. + # This is to prevent that a token which was signed by the correct issuer + # but never intended to be used with Weaviate can be used for authentication. + # + # For more information on what clients are in OAuth 2.0, see + # https://tools.ietf.org/html/rfc6749#section-1.1 + client_id: 'my-weaviate-client' + + # username_claim (required) tells Weaviate which claim in the token to use for extracting + # the username. The username will be passed to the authorization plugin. + username_claim: 'email' + + # skip_client_id_check (optional, defaults to false) skips the client_id + # validation in the audience claim as outlined in the section above. + # Not recommended to set this option as it reduces security, only set this + # if your token issuer is unable to provide a correct audience claim + skip_client_id_check: 'false' + + # scope (optional) these will be used by clients as default scopes for authentication + scopes: '' + + # groups_claim: '' +``` + +### Note: Configuring the OIDC token issuer + +Configuring the OIDC token issuer is outside the scope of this document, but here are a few options as a starting point: + +- For simple use-cases such as for a single user, you can use Weaviate Cloud (WCD) as the OIDC token issuer. To do so: + - Make sure you have a WCD account (you can [sign up here](https://console.weaviate.cloud/)). + - In the Docker Compose file (e.g. `docker-compose.yml`), specify: + - `https://auth.wcs.api.weaviate.io/auth/realms/SeMI` as the issuer (in `AUTHENTICATION_OIDC_ISSUER`), + - `wcs` as the client id (in `AUTHENTICATION_OIDC_CLIENT_ID`), and + - enable the adminlist (`AUTHORIZATION_ADMINLIST_ENABLED: 'true'`) and add your WCD account email as the user (in `AUTHORIZATION_ADMINLIST_USERS`) . + - `email` as the username claim (in `AUTHENTICATION_OIDC_USERNAME_CLAIM`). + +- If you need a more customizable setup you can use commercial OIDC providers like [Okta](https://www.okta.com/). +- As another alternative, you can run your own OIDC token issuer server, which may be the most complex but also configurable solution. Popular open-source solutions include Java-based [Keycloak](https://www.keycloak.org/) and Golang-based [dex](https://github.com/dexidp/dex). + +:::info +By default, Weaviate validates that the token includes a specified client id in the audience claim. If your token issuer does not support this feature, you can turn it off as outlined in the configuration section below. +::: + +## Anonymous Access + +Weaviate can be configured to accept anonymous requests. This is strongly discouraged except for development or evaluation purposes. + +Users that send requests without explicit authentication are authenticated as `user: anonymous`. + +You can use the authorization plugin to specify which permissions to apply to this `anonymous` user. If anonymous access is disabled altogether, any request without an allowed authentication scheme returns `401 Unauthorized`. + +### Anonymous access: Docker + +To enable anonymous access in Docker Compose, add the following environment variable to your configuration file: + +```yaml +services: + weaviate: + ... + environment: + ... + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' +``` + +### Anonymous access: Kubernetes + +To enable anonymous access in Kubernetes, add the following configuration to your `values.yaml` file: + +```yaml +authentication: + anonymous_access: + enabled: true +``` + +## Further resources + +- [Configuration: Authorization and RBAC](./authorization.md) +- [References: Environment variables / Authentication and Authorization](../config-refs/env-vars.md#authentication-and-authorization) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/configuration/authorization.md b/docs/deploy/config-guides/authorization.md similarity index 99% rename from docs/weaviate/configuration/authorization.md rename to docs/deploy/config-guides/authorization.md index fdab62fe..51711aac 100644 --- a/docs/weaviate/configuration/authorization.md +++ b/docs/deploy/config-guides/authorization.md @@ -1,6 +1,5 @@ --- title: Authorization -sidebar_position: 35 image: og/docs/configuration.jpg # tags: ['authorization'] --- diff --git a/docs/weaviate/configuration/backups.md b/docs/deploy/config-guides/backups.md similarity index 96% rename from docs/weaviate/configuration/backups.md rename to docs/deploy/config-guides/backups.md index 3731185a..d2ef9780 100644 --- a/docs/weaviate/configuration/backups.md +++ b/docs/deploy/config-guides/backups.md @@ -1,6 +1,5 @@ --- title: Backups -sidebar_position: 12 image: og/docs/configuration.jpg --- @@ -62,15 +61,6 @@ Restart Weaviate to apply the new configuration. Then, you are ready to start a /> - - - - - - - - - - - - - - - - - - diff --git a/docs/deploy/config-guides/modules.md b/docs/deploy/config-guides/modules.md new file mode 100644 index 00000000..25d48bda --- /dev/null +++ b/docs/deploy/config-guides/modules.md @@ -0,0 +1,152 @@ +--- +title: Modules +image: og/docs/configuration.jpg +# tags: ['configuration', 'modules'] +--- + +Weaviate's functionality can be customized by using [modules](/docs/weaviate/modules/index.md). This page explains how to enable and configure modules. + +## Instance-level configuration + +At the instance (i.e. Weaviate cluster) level, you can: + +- Enable modules +- Configure the default vectorizer module +- Configure module-specific variables (e.g. API keys), where applicable + +This can be done by setting the appropriate [environment variables](/docs/weaviate/config-refs/env-vars.md) as shown below. + +:::tip What about WCD? +Weaviate Cloud (WCD) instances come with modules pre-configured. See [this page](/docs/cloud/manage-clusters/status#enabled-modules) for details. +::: + +### Enable individual modules + +You can enable modules by specifying the list of modules in the `ENABLE_MODULES` variable. For example, this code enables the `text2vec-transformers` module. + +```yaml +services: + weaviate: + environment: + ENABLE_MODULES: 'text2vec-transformers' +``` + +To enable multiple modules, add them in a comma-separated list. + +This example code enables the `'text2vec-huggingface`, `generative-cohere`, and `qna-openai` modules. + +```yaml +services: + weaviate: + environment: + ENABLE_MODULES: 'text2vec-huggingface,generative-cohere,qna-openai' +``` + +### Enable all API-based modules + +:::caution Experimental feature +Available starting in `v1.26.0`. This is an experimental feature. Use with caution. +::: + +You can enable all API-based modules by setting the `ENABLE_API_BASED_MODULES` variable to `true`. This will enable all API-based [model integrations](/docs/weaviate/model-providers/index.md), such as those for Anthropic, Cohere, OpenAI and so on by enabling the relevant modules. These modules are lightweight, so enabling them all will not significantly increase resource usage. + +```yaml +services: + weaviate: + environment: + ENABLE_API_BASED_MODULES: 'true' +``` + +The list of API-based modules can be found on the [model provider integrations page](/docs/weaviate/model-providers/index.md#api-based). You can also inspect the [source code](https://github.com/weaviate/weaviate/blob/main/adapters/handlers/rest/configure_api.go) where the list is defined. + +This can be combined with enabling individual modules. For example, the example below enables all API-based modules, Ollama modules and the `backup-s3` module. + +```yaml +services: + weaviate: + environment: + ENABLE_API_BASED_MODULES: 'true' + ENABLE_MODULES: 'text2vec-ollama,generative-ollama,backup-s3' +``` + +Note that enabling multiple vectorizer (e.g. `text2vec`, `multi2vec`) modules will disable the [`Explore` functionality](/docs/weaviate/api/graphql/explore.md). If you need to use `Explore`, you should only enable one vectorizer module. + +### Module-specific variables + +You may need to specify additional environment variables to configure each module where applicable. For example, the `backup-s3` module requires the backup S3 bucket to be set via `BACKUP_S3_BUCKET`, and the `text2vec-contextionary` module requires the inference API location via `TRANSFORMERS_INFERENCE_API`. + +Refer to the individual [module documentation](../modules/index.md) for more details. + +## Vectorizer modules + +The [vectorization integration](../model-providers/index.md) enable Weaviate to vectorize data at import, and to perform [`near`](/docs/weaviate/search/similarity.md) searches such as `nearText` or `nearImage`. + +:::info List of available vectorizer integrations +Can be found [in this section](/docs/weaviate/model-providers/index.md). +::: + +### Enable vectorizer modules + +You can enable vectorizer modules by adding them to the `ENABLE_MODULES` environment variable. For example, this code enables the `text2vec-cohere`, `text2vec-huggingface`, and `text2vec-openai` vectorizer modules. + +```yaml +services: + weaviate: + environment: + ENABLE_MODULES: 'text2vec-cohere,text2vec-huggingface,text2vec-openai' +``` + +### Default vectorizer module + +You can specify a default vectorization module with the `DEFAULT_VECTORIZER_MODULE` variable as below. + +If a default vectorizer module is not set, you must set a vectorizer in the schema before you can use `near` or vectorization at import time. + +This code sets `text2vec-huggingface` as the default vectorizer. Thus, `text2vec-huggingface` module will be used unless another vectorizer is specified for that class. + +``` yaml +services: + weaviate: + environment: + DEFAULT_VECTORIZER_MODULE: text2vec-huggingface +``` + +## Generative model integrations + +The [generative model integrations](/docs/weaviate/model-providers/index.md) enable [retrieval augmented generation](/docs/weaviate/search/generative.md) functions. + +### Enable a generative module + +You can enable generative modules by adding the desired module to the `ENABLE_MODULES` environment variable. For example, this code enables the `generative-cohere` module and the `text2vec-huggingface` vectorizer module. + +```yaml +services: + weaviate: + environment: + ENABLE_MODULES: 'text2vec-huggingface,generative-cohere' +``` + +:::tip `generative` module selection unrelated to `text2vec` module selection +Your choice of the `text2vec` module does not restrict your choice of `generative` module, or vice versa. +::: + +## Tenant offload modules + +Tenants can be offloaded to cold storage to reduce memory and disk usage, and onloaded back when needed. + +See the [dedicated page on tenant offloading](/docs/weaviate/configuration/tenant-offloading.md) for more information on how to configure Weaviate for tenant offloading. For information on how to offload and onload tenants, see [How-to: manage tenant states](/docs/weaviate/manage-data/tenant-states.mdx). + +## Custom modules + +See [here](/docs/weaviate/concepts/modules.md) how you can create and use your own modules. + +## Related pages + +- [Concepts: Modules](/docs/weaviate/concepts/modules.md) +- [References: Modules](/docs/weaviate/modules/index.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/config-guides/monitoring-obs/grafana-prometheus.md b/docs/deploy/config-guides/monitoring-obs/grafana-prometheus.md new file mode 100644 index 00000000..be198a1c --- /dev/null +++ b/docs/deploy/config-guides/monitoring-obs/grafana-prometheus.md @@ -0,0 +1,8 @@ +--- + +title: Grafana and Prometheus in Production + +--- + +So you've got your Weaviate deployment up and running and have now found yourself wondering about how to collect metrics of your deployment? Fear not, in this document we'll show how to use Grafana agent to scrape your metrics. + diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-async-queue.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-async-queue.png new file mode 100644 index 00000000..903f8aa6 Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-async-queue.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-importing.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-importing.png new file mode 100644 index 00000000..09fceb95 Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-importing.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-lsm.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-lsm.png new file mode 100644 index 00000000..7cd3a9fd Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-lsm.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-objects.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-objects.png new file mode 100644 index 00000000..4b0c148b Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-objects.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-startup.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-startup.png new file mode 100644 index 00000000..aae320f1 Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-startup.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-usage.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-usage.png new file mode 100644 index 00000000..09706dca Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-usage.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-vector.png b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-vector.png new file mode 100644 index 00000000..aa3e28a1 Binary files /dev/null and b/docs/deploy/config-guides/monitoring-obs/img/weaviate-sample-dashboard-vector.png differ diff --git a/docs/deploy/config-guides/monitoring-obs/index.mdx b/docs/deploy/config-guides/monitoring-obs/index.mdx new file mode 100644 index 00000000..3c355582 --- /dev/null +++ b/docs/deploy/config-guides/monitoring-obs/index.mdx @@ -0,0 +1,5 @@ +--- +title: Monitoring and Observability +sidebar_position: 0 +--- + diff --git a/docs/deploy/config-guides/monitoring-obs/monitoring.md b/docs/deploy/config-guides/monitoring-obs/monitoring.md new file mode 100644 index 00000000..4fd4d77e --- /dev/null +++ b/docs/deploy/config-guides/monitoring-obs/monitoring.md @@ -0,0 +1,141 @@ +--- +title: Monitoring +sidebar_position: 50 +image: og/docs/configuration.jpg +# tags: ['configuration', 'operations', 'monitoring', 'observability'] +--- + +Weaviate can expose Prometheus-compatible metrics for monitoring. A standard +Prometheus/Grafana setup can be used to visualize metrics on various +dashboards. + +Metrics can be used to measure request latencies, import +speed, time spent on vector vs object storage, memory usage, application usage, +and more. + +## Configure Monitoring + +### Enable within Weaviate + +To tell Weaviate to collect metrics and expose them in a Prometheus-compatible +format, all that's required is to set the following environment variable: + +```sh +PROMETHEUS_MONITORING_ENABLED=true +``` + +By default, Weaviate will expose the metrics at `:2112/metrics`. You +can optionally change the port to a custom port using the following environment +variable: + +```sh +PROMETHEUS_MONITORING_PORT=3456 +``` + +### Scrape metrics from Weaviate + +Metrics are typically scraped into a time-series database, such as Prometheus. +How you consume metrics depends on your setup and environment. + +The [Weaviate examples repo contains a fully pre-configured setup using +Prometheus, Grafana and some example +dashboards](https://github.com/weaviate/weaviate-examples/tree/main/monitoring-prometheus-grafana). +You can start up a full-setup including monitoring and dashboards with a single +command. In this setup the following components are used: + +* Docker Compose is used to provide a fully-configured setup that can be + started with a single command. +* Weaviate is configured to expose Prometheus metrics as outlined in the + section above. +* A Prometheus instance is started with the setup and configured to scrape + metrics from Weaviate every 15s. +* A Grafana instance is started with the setup and configured to use the + Prometheus instance as a metrics provider. Additionally, it runs a dashboard + provider that contains a few sample dashboards. + +### Multi-tenancy + +When using multi-tenancy, we suggest setting the `PROMETHEUS_MONITORING_GROUP` [environment variable](docs/weaviate/config-refs/env-vars.md) as `true` so that data across all tenants are grouped together for monitoring. + +## Obtainable Metrics + +The list of metrics that are obtainable through Weaviate's metric system is +constantly being expanded. The complete list is in the [`prometheus.go`](https://github.com/weaviate/weaviate/blob/main/usecases/monitoring/prometheus.go) source code file. + +This page describes some noteworthy metrics and their uses. + +Typically metrics are quite granular, as they can always be aggregated later +on. For example if the granularity is "shard", you could aggregate all "shard" +metrics of the same "class" to obtain a class metrics, or aggregate all metrics +to obtain the metric for the entire Weaviate instance. + +| Metric | Description | Labels | Type | +| --- | --- | --- | --- | +| `batch_durations_ms` | Duration of a single batch operation in ms. The `operation` label further defines what operation as part of the batch (e.g. object, inverted, vector) is being used. Granularity is a shard of a class. | `operation`, `class_name`, `shard_name` | Histogram | +| `batch_delete_durations_ms` | Duration of a batch delete in ms. The `operation` label further defines what operation as part of the batch delete is being measured. Granularity is a shard of a class | `class_name`, `shard_name` | Histogram | +| `objects_durations_ms` | Duration of an individual object operation, such as `put`, `delete`, etc. as indicated by the `operation` label, also as part of a batch. The `step` label adds additional precisions to each `operation`. Granularity is a shard of a class. | `class_name`, `shard_name` | Histogram | +| `object_count` | Numbers of objects present. Granularity is a shard of a class | `class_name`, `shard_name` | Gauge | +| `async_operations_running` | Number of currently running async operations. The operation itself is defined through the `operation` label. | `operation`, `class_name`, `shard_name`, `path` | Gauge | +| `lsm_active_segments` | Number of currently present segments per shard. Granularity is shard of a class. Grouped by `strategy`.| `strategy`, `class_name`, `shard_name`, `path` | Gauge | +| `lsm_bloom_filter_duration_ms` | Duration of a bloom filter operation per shard in ms. Granularity is shard of a class. Grouped by `strategy`. | `operation`, `strategy`, `class_name`, `shard_name` | Histogram | +| `lsm_segment_objects` | Number of entries per LSM segment by level. Granularity is shard of a class. Grouped by `strategy` and `level`. | `operation`, `strategy`, `class_name`, `shard_name`, `path`, `level` | Gauge | +| `lsm_segment_size` | Size of LSM segment by level and unit. | `strategy`, `class_name`, `shard_name`, `path`, `level`, `unit` | Gauge | +| `lsm_segment_count` | Number of segments by level | `strategy`, `class_name`, `shard_name`, `path`, `level` | Gauge | +| `vector_index_tombstones` | Number of currently active tombstones in the vector index. Will go up on each incoming delete and go down after a completed repair operation. | `class_name`, `shard_name` | Gauge | +| `vector_index_tombstone_cleanup_threads` | Number of currently active threads for repairing/cleaning up the vector index after deletes have occurred. | `class_name`, `shard_name` | Gauge | +| `vector_index_tombstone_cleaned` | Total number of deleted and removed vectors after repair operations. | `class_name`, `shard_name` | Counter | +| `vector_index_operations` | Total number of mutating operations on the vector index. The operation itself is defined by the `operation` label. | `operation`, `class_name`, `shard_name` | Gauge | +| `vector_index_size` | The total capacity of the vector index. Typically larger than the number of vectors imported as it grows proactively. | `class_name`, `shard_name` | Gauge | +| `vector_index_maintenance_durations_ms` | Duration of a sync or async vector index maintenance operation. The operation itself is defined through the `operation` label. | `opeartion`, `class_name`, `shard_name` | Histogram | +| `vector_index_durations_ms` | Duration of regular vector index operation, such as insert or delete. The operation itself is defined through the `operation` label. The `step` label adds more granularity to each operation. | `operation`, `step`, `class_name`, `shard_name` | Histogram | +| `startup_durations_ms` | Duration of individual startup operations in ms. The operation itself is defined through the `operation` label. | `operation`, `class_name`, `shard_name` | Histogram | +| `startup_diskio_throughput` | Disk I/O throughput in bytes/s at startup operations, such as reading back the HNSW index or recovering LSM segments. The operation itself is defined by the `operation` label. | `operation`, `step`, `class_name`, `shard_name` | Histogram | +| `requests_total` | Metric that tracks all user requests to determine if it was successful or failed. | `api`, `query_type`, `class_name` | `GaugeVec` | +| `index_queue_push_duration_ms` | Duration of pushing one or more vectors to the index queue. | `class_name`, `shard_name`, `target_vector` | `Summary` | +| `index_queue_delete_duration_ms` | Duration of deleting one or more vectors from the index queue and the underlying index. | `class_name`, `shard_name`, `target_vector` | `Summary` | +| `index_queue_preload_duration_ms` | Duration of preloading un-indexed vectors to the index queue. | `class_name`, `shard_name`, `target_vector` | `Summary` | +| `index_queue_preload_count` | Number of vectors preloaded to the index queue. | `class_name`, `shard_name`, `target_vector` | `Gauge` | +| `index_queue_search_duration_ms` | Duration of searching for vectors in the index queue and the underlying index. | `class_name`, `shard_name`, `target_vector` | `Summary` | +| `index_queue_paused` | Whether the index queue is paused. | `class_name`, `shard_name`, `target_vector` | `Gauge` | +| `index_queue_size` | Number of vectors in the index queue. | `class_name`, `shard_name`, `target_vector` | `Gauge` | +| `index_queue_stale_count` | Number of times the index queue has been marked as stale. | `class_name`, `shard_name`, `target_vector` | `Counter` | +| `index_queue_vectors_dequeued` | Number of vectors sent to the workers per tick. | `class_name`, `shard_name`, `target_vector` | `Gauge` | +| `index_queue_wait_duration_ms` | Duration of waiting for the workers to finish. | `class_name`, `shard_name`, `target_vector` | `Summary` | + +Extending Weaviate with new metrics is very easy. To suggest a new metric, see the [contributor guide](/docs/contributor-guide). + +### Versioning + +Be aware that metrics do not follow the semantic versioning guidelines of other Weaviate features. Weaviate's main APIs are stable and breaking changes are extremely rare. Metrics, however, have shorter feature lifecycles. It can sometimes be necessary to introduce an incompatible change or entirely remove a metric, for example, because the cost of observing a specific metric in production has grown too high. As a result, it is possible that a Weaviate minor release contains a breaking change for the Monitoring system. If so, it will be clearly highlighted in the release notes. + +## Sample Dashboards + +Weaviate does not ship with any dashboards by default, but here is a list of +dashboards being used by the various Weaviate teams, both during development, +and when helping users. These do not come with any support, but may still be +helpful. Treat them as inspiration to design your own dashboards which fit +your uses perfectly: + +| Dashboard | Purpose | Preview | +| --- | --- | --- | +| [Importing Data Into Weaviate](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/importing.json) | Visualize speed of import operations (including its components, such as object store, inverted index, and vector index). | ![Importing Data into Weaviate](./img/weaviate-sample-dashboard-importing.png "Importing Data Into Weaviate") | +| [Object Operations](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/objects.json) | Visualize speed of whole object operations, such as GET, PUT, etc. | ![Objects](./img/weaviate-sample-dashboard-objects.png "Objects") | +| [Vector Index](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/vectorindex.json) | Visualize the current state, as well as operations on the HNSW vector index | ![Vector Index](./img/weaviate-sample-dashboard-vector.png "Vector Index") | +| [LSM Stores](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/lsm.json) | Get insights into the internals (including segments) of the various LSM stores within Weaviate. | ![LSM Store](./img/weaviate-sample-dashboard-lsm.png "LSM Store") | +| [Startup](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/startup.json) | Visualize the startup process, including recovery operations | ![Startup](./img/weaviate-sample-dashboard-startup.png "Vector Index") | +| [Usage](https://github.com/weaviate/weaviate/blob/master/tools/dev/grafana/dashboards/usage.json) | Obtain usage metrics, such as number of objects imported, etc.| ![Usage](./img/weaviate-sample-dashboard-usage.png "Usage") | +| [Aysnc index queue](https://github.com/weaviate/weaviate/blob/main/tools/dev/grafana/dashboards/index_queue.json) | Observe index queue activity | ![Async index queue](./img/weaviate-sample-dashboard-async-queue.png "Async index queue") | + +## `nodes` API Endpoint + +To get collection details programmatically, use the [`nodes`](../config-refs/nodes.md) REST endpoint. + +import APIOutputs from '/_includes/rest/node-endpoint-info.mdx'; + + + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/config-refs/nodes.md b/docs/deploy/config-guides/nodes.md similarity index 98% rename from docs/weaviate/config-refs/nodes.md rename to docs/deploy/config-guides/nodes.md index ff4ea134..a4dc1205 100644 --- a/docs/weaviate/config-refs/nodes.md +++ b/docs/deploy/config-guides/nodes.md @@ -1,6 +1,5 @@ --- title: Cluster node data -sidebar_position: 93 image: og/docs/configuration.jpg # tags: ['nodes', 'reference', 'configuration'] --- diff --git a/docs/deploy/config-guides/persistence.md b/docs/deploy/config-guides/persistence.md new file mode 100644 index 00000000..5649bf34 --- /dev/null +++ b/docs/deploy/config-guides/persistence.md @@ -0,0 +1,112 @@ +--- +title: Persistence +image: og/docs/configuration.jpg +--- + +import SkipLink from '/src/components/SkipValidationLink' + +When running Weaviate with Docker or Kubernetes, you can persist its data by mounting a volume to store the data outside of the containers. Doing so will cause the Weaviate instance to also load the data from the mounted volume when it is restarted. + +Note that Weaviate now offers native backup modules starting with `v1.15` for single-node instances, and `v1.16` for multi-node instances. For older versions of Weaviate, persisting data as described here will allow you to back up Weaviate. + +## Docker Compose + +### Persistence + +When running Weaviate with Docker Compose, you can set the `volumes` variable under the `weaviate` service and a unique cluster hostname as an environment variable. + +```yaml +services: + weaviate: + volumes: + - /var/weaviate:/var/lib/weaviate + environment: + CLUSTER_HOSTNAME: 'node1' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' +``` + +* About the volumes + * `/var/weaviate` is the location where you want to store the data on the local machine + * `/var/lib/weaviate` The value after the colon (:) is the storage location inside the container. This value must match the PERSISTENCE_DATA_PATH variable. +* About the hostname + * `CLUSTER_HOSTNAME` can be any name + +In the case you want a more verbose output, you can change the environment variable for the `LOG_LEVEL` + +```yaml +services: + weaviate: + environment: + LOG_LEVEL: 'debug' +``` + +A complete example of a Weaviate without modules but with an externally mounted volume and more verbose output: + +```yaml +--- +services: + weaviate: + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8080:8080 + - 50051:50051 + restart: on-failure:0 + volumes: + - /var/weaviate:/var/lib/weaviate # <== set a volume here + environment: + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node1' # <== this can be set to an arbitrary name +... +``` + +### Backups + +See [Backups](./backups.md). + +## Kubernetes + +For Kubernetes setup, the only thing to bear in mind is that Weaviate needs a `PersistentVolumes` through `PersistentVolumeClaims` ([more info](../installation-guides/k8s-installation.md#requirements)) but the Helm chart is already configured to store the data on an external volume. + +## Disk Pressure Warnings and Limits + +Starting with `v1.12.0` there are two levels of disk usage notifications and actions configured through environment variables. Both variables are optional. If not set they default to the values outlined below: + +| Variable | Default Value | Description | +| --- | --- | --- | +| `DISK_USE_WARNING_PERCENTAGE` | `80` | If disk usage is higher than the given percentage a warning will be logged by all shards on the affected node's disk | +| `DISK_USE_READONLY_PERCENTAGE` | `90` | If disk usage is higher than the given percentage all shards on the affected node will be marked as `READONLY`, meaning all future write requests will fail. | + +If a shard was marked `READONLY` due to disk pressure and you want to mark the +shard as ready again (either because you have made more space available or +changed the thresholds) you can use the Shards API to do so. + +## Disk access method + +:::info Added in `v1.21` +::: + +Weaviate maps data on disk to memory. To configure how Weaviate uses virtual memory, set the `PERSISTENCE_LSM_ACCESS_STRATEGY` environment variable. The default value is `mmap`. Use `pread` to as an alternative. + +The two functions reflect different under-the-hood memory management behaviors. `mmap` uses a memory-mapped file, which means that the file is mapped into the virtual memory of the process. `pread` is a function that reads data from a file descriptor at a given offset. + +In general, `mmap` may be a preferred option with memory management benefits. However, if you experience stalling situations under heavy memory load, we suggest trying `pread` instead. + + +## Related pages +- [Configuration: Backups](docs/weaviate/configuration/backups.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/config-guides/replication.md b/docs/deploy/config-guides/replication.md new file mode 100644 index 00000000..9d09615d --- /dev/null +++ b/docs/deploy/config-guides/replication.md @@ -0,0 +1,120 @@ +--- +title: Replication +image: og/docs/configuration.jpg +# tags: ['configuration', 'operations', 'monitoring', 'observability'] +--- + +Weaviate instances can be replicated. Replication can improve read throughput, improve availability, and enable zero-downtime upgrades. + +For more details on how replication is designed and built in Weaviate, see [Replication Architecture](../concepts/replication-architecture/index.md). + +## How to configure + +import RaftRFChangeWarning from '/\_includes/1-25-replication-factor.mdx'; + + + +Replication is disabled by default. It can be enabled per collection in the [collection configuration](../manage-collections/multi-node-setup.mdx#replication-settings). This means you can set different replication factors per class in your dataset. + +To enable replication, you can set one or both of the following: + +- `REPLICATION_MINIMUM_FACTOR` environment variable for the entire Weaviate instance, or +- `replicationFactor` parameter for a collection. + +### Weaviate-wide minimum replication factor + +The `REPLICATION_MINIMUM_FACTOR` environment variable sets the minimum replication factor for all collections in the Weaviate instance. + +If you set the [replication factor for a collection](#replication-factor-for-a-collection), the collection's replication factor overrides the minimum replication factor. + + +## Data consistency + +When Weaviate detects inconsistent data across nodes, it attempts to repair the out of sync data. + +Starting in v1.26, Weaviate adds [async replication](../concepts/replication-architecture/consistency.md#async-replication) to proactively detect inconsistencies. In earlier versions, Weaviate uses a [repair-on-read](../concepts/replication-architecture/consistency.md#repair-on-read) strategy to repair inconsistencies at read time. + +Repair-on-read is automatic. To activate async replication, set `asyncEnabled` to true in the `replicationConfig` section of your collection definition. + +import ReplicationConfigWithAsyncRepair from '/\_includes/code/configuration/replication-consistency.mdx'; + + + +### Configure async replication settings {#async-replication-settings} + +:::info Added in `v1.29` +The [environment variables](/docs/weaviate/config-refs/env-vars#async-replication) for configuring async replication (`ASYNC_*`) have been introduced in `v1.29`. +::: + +Async replication helps achieve consistency for data replicated across multiple nodes. + +Update the following [environment variables](/docs/weaviate/config-refs/env-vars#async-replication) to configure async replication for your particular use case. + +#### Logging + +- **Set the frequency of the logger:** `ASYNC_REPLICATION_LOGGING_FREQUENCY` + Define how often the async replication background process will log events. + +#### Data comparison + +- **Set the frequency of comparisons:** `ASYNC_REPLICATION_FREQUENCY` + Define how often each node compares its local data with other nodes. +- **Set comparison timeout:** `ASYNC_REPLICATION_DIFF_PER_NODE_TIMEOUT` + Optionally configure a timeout for how long to wait during comparison when a node is unresponsive. +- **Monitor node availability:** `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` + Trigger comparisons whenever there’s a change in node availability. +- **Configure hash tree height:** `ASYNC_REPLICATION_HASHTREE_HEIGHT` + Specify the size of the hash tree, which helps narrow down data differences by comparing hash digests at multiple levels instead of scanning entire datasets. See [this page](../concepts/replication-architecture/consistency.md#memory-and-performance-considerations-for-async-replication) for more information on the memory and performance considerations for async replication. +- **Batch size for digest comparison:** `ASYNC_REPLICATION_DIFF_BATCH_SIZE` + Define the number of objects whose digest (e.g., last update time) is compared between nodes before propagating actual objects. + +#### Data synchronization + +Once differences between nodes are detected, Weaviate propagates outdated or missing data. Configure synchronization as follows: + +- **Set the frequency of propagation:** `ASYNC_REPLICATION_FREQUENCY_WHILE_PROPAGATING` + After synchronization is completed on a node, temporarily adjust the data comparison frequency to the set value. +- **Set propagation timeout:** `ASYNC_REPLICATION_PROPAGATION_TIMEOUT` + Optionally configure a timeout for how long to wait during propagation when a node is unresponsive. +- **Set propagation delay:** `ASYNC_REPLICATION_PROPAGATION_DELAY` + Define a delay period to allow asynchronous write operations to reach all nodes before propagating new or updated objects. +- **Batch size for data propagation:** `ASYNC_REPLICATION_PROPAGATION_BATCH_SIZE` + Define the number of objects that are sent in each synchronization batch during the propagation phase. +- **Set propagation limits:** `ASYNC_REPLICATION_PROPAGATION_LIMIT` + Enforce a limit on the number of out-of-sync objects to be propagated per replication iteration. +- **Set propagation concurrency:** `ASYNC_REPLICATION_PROPAGATION_CONCURRENCY` + Specify the number of concurrent workers that can send batches of objects to other nodes, allowing multiple propagation batches to be sent simultaneously. + +:::tip +Tweak these settings based on your cluster size and network latency to achieve optimal performance. Smaller batch sizes and shorter timeouts may be beneficial for high-traffic clusters, while larger clusters might require more conservative settings. +::: + +## How to use: Queries + +When you add (write) or query (read) data, one or more replica nodes in the cluster will respond to the request. How many nodes need to send a successful response and acknowledgment to the coordinator node depends on the `consistency_level`. Available [consistency levels](../concepts/replication-architecture/consistency.md) are `ONE`, `QUORUM` (replication_factor / 2 + 1) and `ALL`. + +The `consistency_level` can be specified at query time: + +```bash +# Get an object by ID, with consistency level ONE +curl "http://localhost:8080/v1/objects/{ClassName}/{id}?consistency_level=ONE" +``` + +:::note +In v1.17, only [read queries that get data by ID](../manage-objects/read.mdx#get-an-object-by-id) had a tunable consistency level. All other object-specific REST endpoints (read or write) used the consistency level `ALL`. Starting with v1.18, all write and read queries are tunable to either `ONE`, `QUORUM` (default) or `ALL`. GraphQL endpoints use the consistency level `ONE` (in both versions). +::: + +import QueryReplication from '/\_includes/code/replication.get.object.by.id.mdx'; + + + +## Related pages + +- [Concepts: Replication Architecture](../concepts/replication-architecture/index.md) +- [Configurinfg Async Replication](./async-rep.md) + +## Questions and feedback + +import DocsFeedback from '/\_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/config-guides/scaling-strategies/horizontal-scaling.mdx b/docs/deploy/config-guides/scaling-strategies/horizontal-scaling.mdx new file mode 100644 index 00000000..fd20d426 --- /dev/null +++ b/docs/deploy/config-guides/scaling-strategies/horizontal-scaling.mdx @@ -0,0 +1,371 @@ +--- +title: Horizontal Scaling Deployment Strategies +--- + +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import TabItem from '@theme/TabItem'; +import Tabs from '@theme/Tabs'; +import PyCode from '!!raw-loader!/_includes/code/howto/manage-data.collections.py'; +import PyCodeV3 from '!!raw-loader!/_includes/code/howto/manage-data.collections-v3.py'; +import TSCode from '!!raw-loader!/_includes/code/howto/manage-data.collections.ts'; +import TSCodeLegacy from '!!raw-loader!/_includes/code/howto/manage-data.collections-v2.ts'; +import JavaCode from '!!raw-loader!/_includes/code/howto/java/src/test/java/io/weaviate/docs/manage-data.classes.java'; +import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/manage-data.classes_test.go'; + + +This document provides detailed implementation guidance for horizontally scaling your Weaviate deployment. + +## Scaling Methods + +![Sharding vs Replication](./imgs/horiz-scaling.png) + + + +### Replication + +Replication creates redundant copies of your data, it is useful when your data needs to be highly available. + +### Sharding + +Sharding divides data across nodes, it is useful when your dataset is too large for just a single node. + + +### Choosing your strategy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Requirement/GoalShardingReplicationBoth CombinedPrimary Consideration
+ Handle dataset too large for single node + +
Yes
+ 		+
No
+ 		+
Yes
+ 		+
+

How much data are you storing?

+
    +
  • + Vector dimensions and count determine memory requirements +
    • +
  • + Sharding divides this across nodes +
    • +
+
+
+ Improve query throughput + +
Maybe*
+ 		+
Yes
+ 		+
Yes
+ 		+
+

Is your workload read-heavy?

+
    +
  • + Replication allows distributing read queries across nodes +
    • +
  • + Sharding may help with certain query patterns +
    • +
+
+
+ Accelerate data imports + +
Yes
+ 		+
No
+ 		+
Yes
+ 		+
+

Is import speed a priority?

+
    +
  • + Sharding enables parallel processing of imports +
    • +
  • + Replication adds overhead during imports +
    • +
+
+
+ Ensure high availability + +
No
+ 		+
Yes
+ 		+
Yes
+ 		+
+

Can you tolerate downtime?

+
    +
  • + Replication provides redundancy if nodes fail +
    • +
  • + Without replication, shard loss = data loss +
    • +
+
+
+ Enable zero-downtime upgrades + +
No
+ 		+
Yes
+ 		+
Yes
+ 		+
+

How critical is continuous operation?

+
    +
  • + Replication allows rolling updates +
    • +
  • + Production systems typically require this capability +
    • +
+
+
+ Optimize resource utilization + +
Yes
+ 		+
Maybe*
+ 		+
Maybe*
+ 		+
+

Are you resource-constrained?

+
    +
  • + Sharding distributes load efficiently +
    • +
  • + Replication adds resource overhead +
    • +
+
+
+ Geographic distribution + +
No
+ 		+
Yes
+ 		+
Yes
+ 		+
+

Do you need multi-region support?

+
    +
  • + Replicas can be deployed across regions +
    • +
  • + Reduces latency for geographically distributed users +
    • +
+
+
+ +**This may serve as a partial solution and will depend on your configuration.* + +### Sharding: Divide and Conquer + +You've made the decision to shard your data, now let's get it configured: + + + + + + + + + + + + + + + + + +#### Parameters + +These parameters are used to configure your collection shards: + +| Parameter | Type | Description | +| :-------------------- | :------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `desiredCount` | integer | *Immutable, Optional*. Controls the target number of physical shards for the collection index. Defaults to the number of nodes in the cluster, but can be explicitly set lower. If set higher than the node count, some nodes will host multiple shards. | +| `virtualPerPhysical` | integer | *Immutable, Optional*. Defines how many virtual shards correspond to one physical shard, defaulting to `128`. Using virtual shards aids in reducing data movement during resharding. | +| `desiredVirtualCount` | integer | *Read-only*. Shows the target total number of virtual shards, calculated as `desiredCount * virtualPerPhysical`. | + + +### Replication: An army of clones + +Configure your data's replication to ensure it's always available: + +import RaftRFChangeWarning from '/_includes/1-25-replication-factor.mdx'; + + + +Configure replication settings, such as [async replication](/docs/weaviate/configuration/replication#async-replication-settings) and [deletion resolution strategy](../concepts/replication-architecture/consistency.md#deletion-resolution-strategies). + + + + + + + + + + + + + + + +```bash +curl \ +-X POST \ +-H "Content-Type: application/json" \ +-d '{ + "class": "Article", + "properties": [ + { + "dataType": [ + "string" + ], + "description": "Title of the article", + "name": "title" + } + ], + "replicationConfig": { + "factor": 3, + "asyncEnabled": true, + "deletionStrategy": "TimeBasedResolution" + } +}' \ +http://localhost:8080/v1/schema +``` + + + + +#### Sharding and Replication: The Dynamic Duo + +In a highly available environment, combining sharding and replication leverages the power and capabilities of both methods to be a dynamic duo that keeps your deployment highly available. + + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/config-guides/scaling-strategies/imgs/horiz-scaling.png b/docs/deploy/config-guides/scaling-strategies/imgs/horiz-scaling.png new file mode 100644 index 00000000..433be744 Binary files /dev/null and b/docs/deploy/config-guides/scaling-strategies/imgs/horiz-scaling.png differ diff --git a/docs/deploy/config-guides/scaling-strategies/index.mdx b/docs/deploy/config-guides/scaling-strategies/index.mdx new file mode 100644 index 00000000..22d01448 --- /dev/null +++ b/docs/deploy/config-guides/scaling-strategies/index.mdx @@ -0,0 +1,53 @@ +--- + +title: Scaling Overview +sidebar_position: 1 + +--- + +Interested in taking a journey to Weaviate's wonderful world of horizontal scaling and learn about the various scaling strategies available for your deployments? As your database grows from an experimental playground to a mission-critical infrastructure, you'll need strategies to scale effectively. So don't worry, we've got you covered! + +### Why Scaling Matters + +Vector databases face unique scaling challenges and as your data grows: + +- Memory requirements increase dramatically with vector dimensions and dataset size +- Query performance becomes critical as user traffic grows +- System reliability becomes non-negotiable for production deployments +- Cost optimization becomes increasingly important + + +### High Availability: Downtime is not an option + +A highly available architecture ensures that your deployment is operational even when individual components fails, this architecture is essential for: + +- Mission-critical applications where search functionality cannot go offline +- Zero-downtime maintenance for seamless upgrades and updates +- Resilience against hardware failures or cloud provider issues +- Consistent performance even during peak traffic periods + +### Horizontal Scaling: Growing out, not up + +Your deployment will grow beyond the limits of a single machine by distributing your workload across multiple nodes by using horizontal scaling. Implementing this will allow your deployment to: + +- Handle larger datasets that exceed single-machine memory +- Improve query throughput by distributing read operations +- Accelerate imports through parallel processing +- Optimize resource usage across your infrastructure + + +### Next Steps + +- [High Availability Strategies] +- [Horizontal Scaling] + +#### Further resources + +- [Concepts: Horizontal Scaling](/docs/weaviate/concepts/cluster.md) + + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/docs/deploy/config-guides/scaling-strategies/multi-node.md b/docs/deploy/config-guides/scaling-strategies/multi-node.md new file mode 100644 index 00000000..456eaca2 --- /dev/null +++ b/docs/deploy/config-guides/scaling-strategies/multi-node.md @@ -0,0 +1,4 @@ +--- +title: Deploying Highly Available Applications +--- + diff --git a/docs/deploy/faqs/index.md b/docs/deploy/faqs/index.md new file mode 100644 index 00000000..e2fc4e80 --- /dev/null +++ b/docs/deploy/faqs/index.md @@ -0,0 +1,106 @@ +--- + +title: Deployment FAQs + +--- + +Looking for guidance on deploying Weaviate in a production environment? Whether you're scaling up from a proof-of-concept to an enterprise-wide implementation, migrating from another solution, or optimizing for specific workloads, you'll find practical answers to streamline your deployment process. Our goal is to help you achieve reliable performance, maintain data integrity, and minimize operational overhead as you harness the power of vector search for your applications! + + +#### Q1: Why did my cluster suddenly become read-only? + +
+ + Answer + +This almost always means you've run out of disk space. Weaviate protects itself by switching to read-only mode when disk usage exceeds a configured threshold. You'll need to increase the disk size available to Weaviate and then reset the read-only status. + +
+ +#### Q2: How do I deploy Weaviate using AWS Marketplace? + +
+ + Answer + +This [page](../installation-guides/aws-installation.md) will walk you through all the necessary steps to deploy Weaviate using AWS marketplace. + +
+ +#### Q3: How do I deploy Weaviate using GCP Marketplace? + +
+ + Answer + +This [page](../installation-guides/gcp-installation.md) will walk you through all the necessary steps to deploy Weaviate using GCP marketplace. + +
+ +#### Q4: Is there a recommended limit for collections? + +
+ + Answer + +If you plan to create more than **20** collections, you may want to consider multi-tenancy instead for better scaling and performance. + +**Additional information:** [Scaling limits with collections](docs/weaviate/starter-guides/managing-collections/collections-scaling-limits.mdx) + +
+ +#### Q5: What are some common issues that occur during deployment? + +
+ + Answer + +Common issues that occur during deployment include: + +- The cluster becoming `read-only`. +- Query results being inconsistent. +- Nodes unable to maintain consensus. +- Creating too many collections. + +#### Further resources + +For further information, the [troubleshooting page](./troubleshooting.md) will help with some of the common issue encountered. + +
+ +#### Q6: What's the difference between Weaviate and other databases? + +
+ + Answer + +Weaviate has complex processes which means that ingestion and deletion require more steps than other types of databases. Data ingestion tends to take longer than a traditional database because of vectorization and deleting objects can be expensive due to the embedding costs. +
+ +#### Q7: Do my resources free up immediately after deleting objects? + +
+ + Answer + +No, it does not happen instantaneously. When you delete objects a tombstone is created. The data removal and index cleanup occur as background processes. + +
+ +#### Q8: What's the difference between client timeouts and module timeouts? + +
+ + Answer + +- **Client timeout:** These are timeouts between the client and Weaviate server. + +- **Module timeout:** These are timeouts that occurs when Weaviate interacts with external modules like LLMs and vectorizers. + +
+ +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/docs/deploy/faqs/troubleshooting.md b/docs/deploy/faqs/troubleshooting.md new file mode 100644 index 00000000..f62a34e9 --- /dev/null +++ b/docs/deploy/faqs/troubleshooting.md @@ -0,0 +1,65 @@ +--- + +title: Deployment Troubleshooting Guide + +--- + +So you've deployed Weaviate and you're fully immersed in the world of vectors when suddenly you encounter a puzzling mystery. This page will serve as your handbook for when things go awry in "Vector Land!" Consider every error message a clue to solving the mystery you're encountering. + + +## Common issues and solutions + +### The cluster is not accepting new information and there are disk space or `read-only` error messages in the logs. + +
+ +Answer + +#### Identifying the issue +As a first step, you'll want to examine your cluster's logs to identify the problem. If after checking the logs of your cluster you see error messages that include phrases like "read-only" or "disk space," then your cluster is more than likely in a `read-only` state due to insufficient disk space. + +#### Resolving the issue + +To solve this mystery, you'll need to increase the available disk space for your nodes. Once the disk space is increased, then you'll need to manually mark the affected shards or collections as writeable again. +You can also set the [`MEMORY_WARNING_PERCENTAGE`](/weaviate/config-refs/env-vars/index.md#MEMORY_WARNING_PERCENTAGE) environment variable to issue warnings when the memory limit is near. +
+ +### You're receiving inconsistent query results. + +
+ + Answer + +#### Identifying the issue + +To confirm and identify the issue, you'll want to first run the same query multiple times to confirm that the results are inconsistent. If the inconsistent results are persisting, then you probably have asynchronous replication disabled for your deployment. + +#### Resolving the issue + +Check your settings to check if you have asynchronous replication enabled. If `async_replication_disabled` is set to "true" then you'll need to set that variable to "false." Once it is enabled, the logs will show messages that indicate successful peers checks and synchronization for the nodes. +
+ +### Your nodes won't communicate, join a cluster, or maintain consensus. + +
+ + Answer + +#### Identifying the issue + +To confirm and identify the issue, you'll want to first run the same query multiple times to confirm that the results are inconsistent. If the inconsistent results are persisting, then you probably have asynchronous replication disabled for your deployment. + +#### Resolving the issue + +Check your settings to check if you have asynchronous replication enabled. If `async_replication_disabled` is set to "true" then you'll need to set that variable to "false." Once it is enabled, the logs will show messages that indicate successful peers checks and synchronization for the nodes. Additionally, test the [live and ready REST endpoints](/docs/weaviate/api/rest#tag/well-known/GET/.well-known/live) and check the network configuration of the nodes. +
+ +## Further reading + +- [LOG_LEVEL](insert env vars `log_level`) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + \ No newline at end of file diff --git a/docs/deploy/index.mdx b/docs/deploy/index.mdx index 37231681..d88d5afc 100644 --- a/docs/deploy/index.mdx +++ b/docs/deploy/index.mdx @@ -1,52 +1,98 @@ --- -title: Deployment overview +title: Overview description: Deployment overview page sidebar_position: 0 --- -# Tacetve a gemit quisquam praestat meruisse gradientis +import CardsSection from "/src/components/CardsSection"; +import DeploymentCards from "/src/components/DeploymentCards"; +import styles from "/src/components/CardsSection/styles.module.scss"; -## Nec sum vagatur nupta tantus +# Deploying Weaviate -Lorem markdownum pugnando ventoque et ducta et et adurat locus inde cervos -refeci Phaethon clivum linguaque Salmacis geras coeperat mollia. Leucothoen Ceyx -est viret nostraque; meum utque latens -quae, patrios plebe inpositum: torvum. **Stipite fluminaque videri**; ictu -duabus sua *condita monstrum*. Imo undis: aut niveo sidoniae; placato sorores -at, quae perque. *Candore dolore*, et secum rastra ceratis et sollicitat vincula -matri mei alios in rigore murmur foedataque, cum. +:::info[Section under construction!] -Et solvit ex comitante super luctus. Unda posse perque venitque quod algae -lymphis postquam optime. +This is a new section being created, please bear with us as we add in new content! -## Flammis levi harena illi membra caudam +::: -Ipse inducitur stridore Aiacem intonsum! Est arma, hausitque *tu* litora -aetasque, at fuit Aegea vides. Non laborum longi fortisque *putes bracchia* -umbras te fingit enim manum! +Weaviate offers multiple deployment options to satisfy your specific use case in production. -Ingentia sive Idaeo serpere, dives inpia. Oculos in aras ergo bracchiaque vera, -omnis solutus vellet: et. Est et putat nec obvia unda -totidem amisit *coniurataeque sic arabat* mea. Caune -quodque **addita** truncis quid latebra: incubat pallorque turribus et ignarus -signum pectore quicquid Abantiades locus. **Ante** aliqua Aiacis deferre, ferunt -hunc omnia, et hic Aconteus nec. +This section hosts common deployment topics, including Kubernetes, cloud providers, and best practices, along with detailed tutorials and how-to guides. +Weaviate is designed for: -Odit festas dixit et quas fidissima, *est umbrae matrona* palude. Ait et quod -pectora circa. +- **Scalability** – Handle billions of vector data points efficiently. +- **High-Performance Search** – Power AI applications with real-time vector retrieval. +- **Flexible Integrations** – Connect with various machine learning models and data sources. +- **Cloud & On-Prem Deployment** – Deploy on Weaviate Cloud, Kubernetes, or managed cloud services. -## Elegit sim ulla poteramus stridente cessit auras +## Deployment Options -Tanta *Elinque*. Mea quae cultum Palameden Aglauros si sacra miserere quam ego, -**e scires** imo sonitusque tumidarum fremit quae ictu? Tum est sub vate vasto, -Galanthida in potui huc et temptat Cenaeo praeterque septem caede. +Choose the best deployment method based on your needs: -> Suo ipse cubitique et vento in onerosa adunco, non. **In paelice Quirites** -> sunt stant sinebat superstes differt non per potentia illiusque nostris -> Hecateidos? Tempore haud imagine facta ora, uni quam fatorum. +export const deploymentCardsData = [ + { + tabs: [ + { label: "Evaluation", active: true }, + { label: "Deployment", active: true }, + { label: "Production", active: true }, + ], + header: "Weaviate Cloud", + bgImage: "/img/site/hex-weaviate.svg", + bgImageLight: "/img/site/hex-weaviate-light.svg", + listItems: [ + "From evaluation (sandbox) to production", + "Serverless (infrastructure managed by Weaviate)", + "(Optional) Data replication (high-availability)", + "(Optional) Zero-downtime updates", + ], + button: { + text: "Set up a WCD instance", + link: "/docs/cloud/manage-clusters/create", + }, + }, + { + tabs: [ + { label: "Evaluation", active: true }, + { label: "Deployment", active: true }, + { label: "Production", active: false }, + ], + header: "Docker", + bgImage: "/img/site/docker-doc-icon.svg", + bgImageLight: "/img/site/docker-doc-icon-light.svg", + listItems: [ + "For local evaluation & development", + "Local inference containers", + "Multi-modal models", + "Customizable configurations", + ], + button: { + text: "Deploy Weaviate with Docker", + link: "/docs/deploy/installation-guides/docker-installation", + }, + }, + { + tabs: [ + { label: "Evaluation", active: false }, + { label: "Deployment", active: true }, + { label: "Production", active: true }, + ], + header: "Kubernetes", + bgImage: "/img/site/kubernetes-doc-icon.svg", + bgImageLight: "/img/site/kubernetes-doc-icon-light.svg", + listItems: [ + "From development to production", + "Local inference containers", + "Multi-modal models", + "Customizable configurations", + "Self-deploy or Marketplace deployment", + "(Optional) Zero-downtime updates", + ], + button: { + text: "Deploy Weaviate with Kubernetes", + link: "/docs/deploy/installation-guides/k8s-installation", + }, + }, +]; -Altis animalibus pampinus venit mirabilis busto: tamen hunc, **salutant** -Stheneleia vultibus pectore. Evocat posuere anxia. - -Aper est; in tuum medius, responderat tamen: magis vix! Amor regno tamen tuli -remige facitote **reverentia**: loqui vade pignora. + \ No newline at end of file diff --git a/docs/deploy/installation-guides/aws-cli.md b/docs/deploy/installation-guides/aws-cli.md new file mode 100644 index 00000000..2df72178 --- /dev/null +++ b/docs/deploy/installation-guides/aws-cli.md @@ -0,0 +1,165 @@ +--- +title: EKS Installation +description: Deploy Weaviate on EKS using the AWS CLI +--- + +Weaviate can be deployed on an EKS cluster using the `eksctl` command-line tool that creates and manages clusters. By the end of this document, you'll have all the necessary information to create an EKS cluster using the command line, add persistent storage to your cluster and then deploy Weaviate onto the cluster. + +:::info Prerequisites + +- Helm installed +- The AWS CLI installed with the latest version +- `kubectl` installed +- `eksctl` installed +::: + +#### Verify your tools + +Before starting, ensure that your tools are installed: + +```bash +helm version +aws --version +kubectl version +eksctl version +``` + +### Step 1: Create the Cluster + +To create your cluster, prepare a `yaml` file that with a name of your choosing (e.g. `eks-cluster.yaml`) + +```bash +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: 'your-cluster-name' # Replace with your desired cluster name + region: 'your-region' # Replace with your desired AWS region + version: "1.33" # Kubernetes version + +iam: + withOIDC: true # Enable IAM OIDC provider + + serviceAccounts: + - metadata: + name: aws-load-balancer-controller + namespace: kube-system + wellKnownPolicies: + awsLoadBalancerController: true + +managedNodeGroups: + - name: node-group-name + labels: { role: worker } + instanceType: t3.large # Choose an appropriate instance type + desiredCapacity: 3 # Number of nodes + minSize: 2 # Minimum number of nodes for autoscaling + maxSize: 5 # Maximum number of nodes for autoscaling + privateNetworking: true # Use private networking + volumeSize: 80 # Root volume size in GB + volumeType: gp3 # Root volume type + +addons: + - name: vpc-cni + version: latest + attachPolicyARNs: + - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + - name: coredns + version: latest + - name: kube-proxy + version: latest + - name: aws-ebs-csi-driver + version: latest + wellKnownPolicies: + ebsCSIController: true # Enable EBS CSI driver +``` + +This creates an EKS cluster within your specified region with an autoscaling node group. There are 3 nodes for high availability and having autoscaling enabled allows for the cluster to dynamically adjust resources based on demand. + +Run this command to create your EKS cluster: +``` +eksctl create cluster -f your-file-name.yaml +``` + +#### Enable `kubectl` to interact with the newly created cluster: + +```bash +aws eks --region update-kubeconfig --name +``` + +#### Verify that the cluster has been created and that you are able to interact with it: + +```bash +kubectl get nodes +``` + +#### Step 2: Add Storage Class + +After creating your cluster and verifying that you can interact with it , you'll need to create a `storageclass.yaml` file: +``` +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: +provisioner: ebs.csi.aws.com +parameters: + type: gp3 + encrypted: "true" +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +``` + +After creating the storage class, apply it: ```kubectl apply -f .yaml``` + + +#### Verify your storage class and has been created and applied + +``` +kubectl get sc +``` + +### Step 3: Add Weaviate to EKS + +After adding persistent storage to your cluster, you can now deploy Weaviate into it. + +#### Create a Weaviate namespace: + +```bash +kubectl create namespace weaviate +``` + +#### Add the Weaviate Helm chart: + +```bash +helm repo add weaviate https://weaviate.github.io/weaviate-helm +helm repo update +``` + +After you've added the Weaviate Helm chart, configure the `values.yaml` file before you deploy Weaviate on the cluster. + + +#### Deploy Weaviate on your cluster: + +```bash +helm upgrade --install weaviate weaviate/weaviate \ + --namespace weaviate \ + --values values.yaml \ +``` + +#### Verify your deployment + +```bash +kubectl get pods +``` + + +### Next Steps: [Connecting to Weaviate](docs/weaviate/connections/index.mdx) + +### Further Resources + +- [Persistent storage for Kubernetes](https://aws.amazon.com/blogs/storage/persistent-storage-for-kubernetes/) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/installation-guides/aws-installation.md b/docs/deploy/installation-guides/aws-installation.md new file mode 100644 index 00000000..185efc58 --- /dev/null +++ b/docs/deploy/installation-guides/aws-installation.md @@ -0,0 +1,61 @@ +--- +title: AWS Marketplace +description: Install Weaviate through the AWS Marketplace for quick cloud deployment. +image: og/docs/installation.jpg +# tags: ['installation', 'AWS Marketplace'] +--- + +import ReactPlayer from 'react-player/lazy' + + + +Through the AWS Marketplace you can launch a Weaviate serverless instance billed directly from AWS. + +:::info Prerequisites +- An AWS account with sufficient credit / payment method. +- (Recommended) Familiarity with AWS and the AWS console. +::: + +You can use [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-ng2dfhb4yjoic?sr=0-3&ref_=beagle&applicationId=AWSMPContessa) to launch a Weaviate serverless instance. + +## Installation instructions + +1. Go to Weaviate's [AWS Marketplace listing](https://aws.amazon.com/marketplace/pp/prodview-ng2dfhb4yjoic?sr=0-3&ref_=beagle&applicationId=AWSMPContessa). +1. Subscribe to the product by following the instructions on the page. + 1. Click View Purchase Options, then go to the next page. + 2. Review the pricing, terms and conditions, and click Subscribe. +From there you will be prompted to set up your account on [Weaviate Cloud](docs/cloud/index.mdx). + +:::info +
+ + A little background information + +- When you deploy Weaviate Serverless Cloud through the AWS Marketplace, you're subscribing to a Software as a Service (SaaS) solution that is specifically built for AWS customers. + +- AWS will notify you once your Weaviate serverless cluster is available. + +**This solution is ideal for:** + +- Organizations requiring AWS billing integration. +- Organizations with regulatory requirements who need specific regional deployments. + +
+ +::: + +## Billing + +You will be charged for Weaviate directly by AWS. + +:::warning + +If you cancel your Weaviate AWS marketplace subscription, your Weaviate organization and its clusters will be deleted by Weaviate. + +::: + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/installation-guides/docker-installation.md b/docs/deploy/installation-guides/docker-installation.md new file mode 100644 index 00000000..7191ec9f --- /dev/null +++ b/docs/deploy/installation-guides/docker-installation.md @@ -0,0 +1,438 @@ +--- +title: Docker +image: og/docs/installation.jpg +# tags: ['installation', 'Docker'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Weaviate supports deployment with Docker. + +You can [run Weaviate with default settings from a command line](#run-weaviate-with-default-settings), or [customize your configuration](#customize-your-weaviate-configuration) by creating your own `docker-compose.yml` file. + +## Run Weaviate with default settings + +:::info Added in v1.24.1 + +::: + +To run Weaviate with Docker using default settings, run this command from from your shell: + +```bash +docker run -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| +``` + +The command sets the following default [environment variables](#environment-variables) in the container: + +- `PERSISTENCE_DATA_PATH` defaults to `./data` +- `AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED` defaults to `true`. +- `QUERY_DEFAULTS_LIMIT` defaults to `10`. + +## Customize your Weaviate configuration + +You can customize your Weaviate configuration by creating a `docker-compose.yml` file. Start from our [sample Docker Compose file](#sample-docker-compose-file), or use the interactive [Configurator](#configurator) to generate a `docker-compose.yml` file. + +## Sample Docker Compose file + +This starter Docker Compose file allows: +* Use of any [API-based model provider integrations](../model-providers/index.md) (e.g. `OpenAI`, `Cohere`, `Google`, and `Anthropic`). + * This includes the relevant embedding model, generative, and reranker [integrations](../model-providers/index.md). +* Searching pre-vectorized data (without a vectorizer). +* Mounts a persistent volume called `weaviate_data` to `/var/lib/weaviate` in the container to store data. + +### Download and run + + + + +Save the code below as `docker-compose.yml` to download and run Weaviate with anonymous access enabled: + +```yaml +--- +services: + weaviate: + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8080:8080 + - 50051:50051 + volumes: + - weaviate_data:/var/lib/weaviate + restart: on-failure:0 + environment: + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node1' +volumes: + weaviate_data: +... +``` + +:::caution +Anonymous access is strongly discouraged except for development or evaluation purposes. +::: + + + + +Save the code below as `docker-compose.yml` to download and run Weaviate with authentication (non-anonymous access) and authorization enabled: + +```yaml +--- +services: + weaviate: + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8080:8080 + - 50051:50051 + volumes: + - weaviate_data:/var/lib/weaviate + restart: on-failure:0 + environment: + QUERY_DEFAULTS_LIMIT: 25 + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node1' + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false' + AUTHENTICATION_APIKEY_ENABLED: 'true' + AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'user-a-key,user-b-key' + AUTHENTICATION_APIKEY_USERS: 'user-a,user-b' + AUTHORIZATION_ENABLE_RBAC: 'true' + AUTHORIZATION_RBAC_ROOT_USERS: 'user-a' +volumes: + weaviate_data: +... +``` + +This setup enables API-key based [authentication](/docs/weaviate/configuration/authentication.md) and role-based access control [authorization](/docs/weaviate/configuration/authorization.md). + +It defines the users `user-a` and `user-b` and corresponding keys `user-a-key` and `user-b-key` which serve as authentication credentials for connecting to your Weaviate instance. + +The user `user-a` is granted admin access rights using the **Role-based access control (RBAC)** method. A custom role can be assigned to the user `user-b` by following the [authorization and RBAC guide](/docs/weaviate/configuration/authorization.md). + + + + +Edit the `docker-compose.yml` file to suit your needs. You can add or remove [environment variables](#environment-variables), change the port mappings, or add additional [model provider integrations](/docs/weaviate/model-providers/index.md), such as [Ollama](/docs/weaviate/model-providers/ollama/index.md), or [Hugging Face Transformers](/docs/weaviate/model-providers/transformers/index.md). + +To start your Weaviate instance, run this command from your shell: + +```bash +docker compose up -d +``` + +## Configurator + +The Configurator can generate a `docker-compose.yml` file for you. Use the Configurator to select specific Weaviate modules, including vectorizers that run locally (i.e. `text2vec-transformers`, or `multi2vec-clip`) + +import DocsConfigGen from '@site/src/components/DockerConfigGen'; + + + +## Environment variables + +You can use environment variables to control your Weaviate setup, authentication and authorization, module settings, and data storage settings. + +:::info List of environment variables +A comprehensive of list environment variables [can be found on this page](/docs/weaviate/config-refs/env-vars.md). +::: + +## Example configurations + +Here are some examples of how to configure `docker-compose.yml`. + +### Persistent volume + +We recommended setting a persistent volume to avoid data loss as well as to improve reading and writing speeds. + +Make sure to run `docker compose down` when shutting down. This writes all the files from memory to disk. + +**With named volume** +```yaml +services: + weaviate: + volumes: + - weaviate_data:/var/lib/weaviate + # etc + +volumes: + weaviate_data: +``` + +After running a `docker compose up -d`, Docker will create a named volume `weaviate_data` and mount it to the `PERSISTENCE_DATA_PATH` inside the container. + +**With host binding** +```yaml +services: + weaviate: + volumes: + - /var/weaviate:/var/lib/weaviate + # etc +``` + +After running a `docker compose up -d`, Docker will mount `/var/weaviate` on the host to the `PERSISTENCE_DATA_PATH` inside the container. + +### Weaviate without any modules + +An example Docker Compose setup for Weaviate without any modules can be found below. In this case, no model inference is performed at either import or search time. You will need to provide your own vectors (e.g. from an outside ML model) at import and search time: + +```yaml +services: + weaviate: + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8080:8080 + - 50051:50051 + restart: on-failure:0 + environment: + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + CLUSTER_HOSTNAME: 'node1' +``` + +### Weaviate with the `text2vec-transformers` module + +An example Docker Compose file with the transformers model [`sentence-transformers/multi-qa-MiniLM-L6-cos-v1`](https://huggingface.co/sentence-transformers/multi-qa-MiniLM-L6-cos-v1) is: + +```yaml +services: + weaviate: + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + restart: on-failure:0 + ports: + - 8080:8080 + - 50051:50051 + environment: + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: "./data" + DEFAULT_VECTORIZER_MODULE: text2vec-transformers + ENABLE_MODULES: text2vec-transformers + TRANSFORMERS_INFERENCE_API: http://t2v-transformers:8080 + CLUSTER_HOSTNAME: 'node1' + t2v-transformers: + image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 + environment: + ENABLE_CUDA: 0 # set to 1 to enable + # NVIDIA_VISIBLE_DEVICES: all # enable if running with CUDA +``` + +Note that transformer models are neural networks built to run on GPUs. Running Weaviate with the `text2vec-transformers` module and without GPU is possible, but it will be slower. Enable CUDA with `ENABLE_CUDA=1` if you have a GPU available. + +For more information on how to set up the environment with the +`text2vec-transformers` integration, see [this +page](/docs/weaviate/model-providers/transformers/embeddings.md). + +The `text2vec-transformers` module requires at least Weaviate version `v1.2.0`. + +### Unreleased versions + +import RunUnreleasedImages from '/_includes/configuration/run-unreleased.mdx' + + + +## Multi-node configuration + +To configure Weaviate to use multiple host nodes, follow these steps: + +- Configure one node as a "founding" member +- Set the `CLUSTER_JOIN` variable for the other nodes in the cluster. +- Set the `CLUSTER_GOSSIP_BIND_PORT` for each node. +- Set the `CLUSTER_DATA_BIND_PORT` for each node. +- Set the `RAFT_JOIN` each node. +- Set the `RAFT_BOOTSTRAP_EXPECT` for each node with the number of voters. +- Optionally, set the hostname for each node using `CLUSTER_HOSTNAME`. + +(Read more about [horizontal replication in Weaviate](/docs/weaviate/concepts/cluster.md).) + +So, the Docker Compose file includes environment variables for the "founding" member that look like this: + +```yaml + weaviate-node-1: # Founding member service name + ... # truncated for brevity + environment: + CLUSTER_HOSTNAME: 'node1' + CLUSTER_GOSSIP_BIND_PORT: '7100' + CLUSTER_DATA_BIND_PORT: '7101' + RAFT_JOIN: 'node1,node2,node3' + RAFT_BOOTSTRAP_EXPECT: 3 +``` + +And the other members' configurations may look like this: + +```yaml + weaviate-node-2: + ... # truncated for brevity + environment: + CLUSTER_HOSTNAME: 'node2' + CLUSTER_GOSSIP_BIND_PORT: '7102' + CLUSTER_DATA_BIND_PORT: '7103' + CLUSTER_JOIN: 'weaviate-node-1:7100' # This must be the service name of the "founding" member node. + RAFT_JOIN: 'node1,node2,node3' + RAFT_BOOTSTRAP_EXPECT: 3 +``` + +Below is an example configuration for a 3-node setup. You may be able to test [replication](/docs/weaviate/configuration/replication.md) examples locally using this configuration. + + +
+ Docker Compose file for a replication setup with 3 nodes + +```yaml +services: + weaviate-node-1: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8080:8080 + - 6060:6060 + - 50051:50051 + restart: on-failure:0 + volumes: + - ./data-node-1:/var/lib/weaviate + environment: + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node1' + CLUSTER_GOSSIP_BIND_PORT: '7100' + CLUSTER_DATA_BIND_PORT: '7101' + RAFT_JOIN: 'node1,node2,node3' + RAFT_BOOTSTRAP_EXPECT: 3 + + weaviate-node-2: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8081:8080 + - 6061:6060 + - 50052:50051 + restart: on-failure:0 + volumes: + - ./data-node-2:/var/lib/weaviate + environment: + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node2' + CLUSTER_GOSSIP_BIND_PORT: '7102' + CLUSTER_DATA_BIND_PORT: '7103' + CLUSTER_JOIN: 'weaviate-node-1:7100' + RAFT_JOIN: 'node1,node2,node3' + RAFT_BOOTSTRAP_EXPECT: 3 + + weaviate-node-3: + init: true + command: + - --host + - 0.0.0.0 + - --port + - '8080' + - --scheme + - http + image: cr.weaviate.io/semitechnologies/weaviate:||site.weaviate_version|| + ports: + - 8082:8080 + - 6062:6060 + - 50053:50051 + restart: on-failure:0 + volumes: + - ./data-node-3:/var/lib/weaviate + environment: + LOG_LEVEL: 'debug' + QUERY_DEFAULTS_LIMIT: 25 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: '/var/lib/weaviate' + ENABLE_API_BASED_MODULES: 'true' + CLUSTER_HOSTNAME: 'node3' + CLUSTER_GOSSIP_BIND_PORT: '7104' + CLUSTER_DATA_BIND_PORT: '7105' + CLUSTER_JOIN: 'weaviate-node-1:7100' + RAFT_JOIN: 'node1,node2,node3' + RAFT_BOOTSTRAP_EXPECT: 3 +``` + +
+ +:::note Port number conventions +It is a Weaviate convention to set the `CLUSTER_DATA_BIND_PORT` to 1 higher than `CLUSTER_GOSSIP_BIND_PORT`. +::: + + +## Shell attachment options + +The output of `docker compose up` is quite verbose as it attaches to the logs of all containers. + +You can attach the logs only to Weaviate itself, for example, by running the following command instead of `docker compose up`: + +```bash +# Run Docker Compose +docker compose up -d && docker compose logs -f weaviate +``` + +Alternatively you can run docker compose entirely detached with `docker compose up -d` _and_ then poll `{bindaddress}:{port}/v1/meta` until you receive a status `200 OK`. + + + +## Troubleshooting + +### Set `CLUSTER_HOSTNAME` if it may change over time + +In some systems, the cluster hostname may change over time. This is known to create issues with a single-node Weaviate deployment. To avoid this, set the `CLUSTER_HOSTNAME` environment variable in the `values.yaml` file to the cluster hostname. + +```yaml +--- +services: + weaviate: + # ... + environment: + CLUSTER_HOSTNAME: 'node1' +... +``` + +## Related pages + +- If you are new to Docker, see [Docker Introduction for Weaviate Users](https://weaviate.io/blog/docker-and-containers-with-weaviate). + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/installation-guides/gcp-installation.md b/docs/deploy/installation-guides/gcp-installation.md new file mode 100644 index 00000000..c3c6f808 --- /dev/null +++ b/docs/deploy/installation-guides/gcp-installation.md @@ -0,0 +1,55 @@ +--- +title: GCP Marketplace +description: Set up Weaviate using Google Cloud Marketplace for simplified deployment. +image: og/docs/installation.jpg +tags: ['installation', 'Google Cloud Marketplace'] +--- + +A Weaviate cluster is easy to deploy with Google Cloud Marketplace (GCP). + +:::info Prerequisites + +- A Google Cloud account with sufficient credit / payment method. +- (Recommended) Familiarity with Google Cloud and the Google Cloud console. +::: + +## Installation instructions + +1. Go to Weaviate's [Google Cloud Marketplace listing](https://console.cloud.google.com/marketplace/product/weaviate-gcp-mktplace/weaviate) page and click Subscribe. +1. Configure and deploy Weaviate by following the on-screen instructions. + +Once completed, you will have a [Weaviate serverless cloud](docs/cloud/index.mdx) deployment. + +:::info +
+ + A little background information + +- When you deploy Weaviate Serverless Cloud through the GCP Marketplace, you're subscribing to a Software as a Service (SaaS) solution that is specifically built for GCP customers. + +- GCP will notify you once your Weaviate serverless cluster is available. + +**This solution is ideal for:** + +- Organizations requiring GCP billing integration. +- Organizations with regulatory requirements who need specific regional deployments. + +
+ +::: + +## Billing + +You will be charged for Weaviate and associated resources directly by Google Cloud. + +:::warning + +If you cancel your Weaviate GCP marketplace subscription, your Weaviate organization and its clusters will be deleted by Weaviate. + +::: + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/deploy/installation-guides/k8s-installation.md b/docs/deploy/installation-guides/k8s-installation.md new file mode 100644 index 00000000..c9b43685 --- /dev/null +++ b/docs/deploy/installation-guides/k8s-installation.md @@ -0,0 +1,254 @@ +--- +title: Kubernetes +sidebar_position: 3 +image: og/docs/installation.jpg +# tags: ['installation', 'Kubernetes'] +--- + +:::tip End-to-end guide +For a tutorial on how to use [minikube](https://minikube.sigs.k8s.io/docs/) to deploy Weaviate on Kubernetes, see the Weaviate Academy course, [Weaviate on Kubernetes](../../academy/deployment/k8s/index.md). +::: + +## Requirements + +* A recent Kubernetes Cluster (at least version 1.23). If you are in a development environment, consider using the kubernetes cluster that is built into Docker desktop. For more information, see the [Docker documentation](https://docs.docker.com/desktop/kubernetes/). +* The cluster needs to be able to provision `PersistentVolumes` using Kubernetes' `PersistentVolumeClaims`. +* A file system that can be mounted read-write by a single node to allow Kubernetes' `ReadWriteOnce` access mode. +* Helm version v3 or higher. The current Helm chart is version `||site.helm_version||`. + +## Weaviate Helm chart + +:::note Important: Set the correct Weaviate version +As a best practice, explicitly set the Weaviate version in the Helm chart.

+ +Set the version in your `values.yaml` file or [overwrite the default value](#deploy-install-the-helm-chart) during deployment. +::: + +To install the Weaviate chart on your Kubernetes cluster, follow these steps: + +### Verify tool setup and cluster access + +```bash +# Check if helm is installed +helm version +# Make sure `kubectl` is configured correctly and you can access the cluster. +# For example, try listing the pods in the currently configured namespace. +kubectl get pods +``` + +### Get the Helm Chart + +Add the Weaviate helm repo that contains the Weaviate helm chart. + +```bash +helm repo add weaviate https://weaviate.github.io/weaviate-helm +helm repo update +``` + +Get the default `values.yaml` configuration file from the Weaviate helm chart: +```bash +helm show values weaviate/weaviate > values.yaml +``` + +### Modify values.yaml + +To customize the Helm chart for your environment, edit the [`values.yaml`](https://github.com/weaviate/weaviate-helm/blob/master/weaviate/values.yaml) +file. The default `yaml` file is extensively documented to help you configure your system. + +#### Replication + +The default configuration defines one Weaviate replica cluster. + +#### Local models + +Local models, such as `text2vec-transformers`, `qna-transformers`, and `img2vec-neural` are disabled by default. To enable a model, set the model's +`enabled` flag to `true`. + +#### Resource limits + +Starting in Helm chart version 17.0.1, constraints on module resources are commented out to improve performance. To constrain resources for specific modules, add the constraints in your `values.yaml` file. + +#### gRPC service configuration + +Starting in Helm chart version 17.0.0, the gRPC service is enabled by default. If you use an older Helm chart, edit your `values.yaml` file to enable gRPC. + +Check that the `enabled` field is set to `true` and the `type` field to `LoadBalancer`. These settings allow you to access the [gRPC API](https://weaviate.io/blog/grpc-performance-improvements) from outside the Kubernetes cluster. + +```yaml +grpcService: + enabled: true # ⬅️ Make sure this is set to true + name: weaviate-grpc + ports: + - name: grpc + protocol: TCP + port: 50051 + type: LoadBalancer # ⬅️ Set this to LoadBalancer (from NodePort) +``` + +#### Authentication and authorization + +An example configuration for authentication is shown below. + +```yaml +authentication: + apikey: + enabled: true + allowed_keys: + - readonly-key + - secr3tk3y + users: + - readonly@example.com + - admin@example.com + anonymous_access: + enabled: false + oidc: + enabled: true + issuer: https://auth.wcs.api.weaviate.io/auth/realms/SeMI + username_claim: email + groups_claim: groups + client_id: wcs +authorization: + admin_list: + enabled: true + users: + - someuser@weaviate.io + - admin@example.com + readonly_users: + - readonly@example.com +``` + +In this example, the key `readonly-key` will authenticate a user as the `readonly@example.com` identity, and `secr3tk3y` will authenticate a user as `admin@example.com`. + +OIDC authentication is also enabled, with WCD as the token issuer/identity provider. Thus, users with WCD accounts could be authenticated. This configuration sets `someuser@weaviate.io` as an admin user, so if `someuser@weaviate.io` were to authenticate, they will be given full (read and write) privileges. + +For further, general documentation on authentication and authorization configuration, see: +- [Authentication](../configuration/authentication.md) +- [Authorization](../configuration/authorization.md) + +#### Run as non-root user + +By default, weaviate runs as the root user. To run as a non-privileged user, edit the settings in the `containerSecurityContext` section. + +The `init` container always runs as root to configure the node. Once the system is started, it run a non-privileged user if you have one configured. + +### Deploy (install the Helm chart) + +You can deploy the helm charts as follows: + +```bash +# Create a Weaviate namespace +kubectl create namespace weaviate + +# Deploy +helm upgrade --install \ + "weaviate" \ + weaviate/weaviate \ + --namespace "weaviate" \ + --values ./values.yaml +``` + +The above assumes that you have permissions to create a new namespace. If you +have only namespace-level permissions, you can skip creating a new +namespace and adjust the namespace argument on `helm upgrade` according to the +name of your pre-configured namespace. + +Optionally, you can provide the `--create-namespace` parameter which will create the namespace if not present. + +### Updating the installation after the initial deployment + +The above command (`helm upgrade...`) is idempotent. In other words, you can run it multiple times after adjusting your desired configuration without causing any unintended changes or side effects. + +### Upgrading to `1.25` or higher from pre-`1.25` + +:::caution Important +::: + +To upgrade to `1.25` or higher from a pre-`1.25` version, you must delete the deployed `StatefulSet`, update the helm chart to version `17.0.0` or higher, and re-deploy Weaviate. + +See the [1.25 migration guide for Kubernetes](../more-resources/migration/weaviate-1-25.md) for more details. + +## Additional Configuration Help + +- [Cannot list resource "configmaps" in API group when deploying Weaviate k8s setup on GCP](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) +- [Error: UPGRADE FAILED: configmaps is forbidden](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) + +### Using EFS with Weaviate + +In some circumstances, you may wish, or need, to use EFS (Amazon Elastic File System) with Weaviate. And we note in the case of AWS Fargate, you must create the [PV (persistent volume)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) manually, as the PVC will NOT create a PV for you. + +To use EFS with Weaviate, you need to: + +- Create an EFS file system. +- Create an EFS access point for every Weaviate replica. + - All of the Access Points must have a different root-directory so that Pods do not share the data, otherwise it will fail. +- Create EFS mount targets for each subnet of the VPC where Weaviate is deployed. +- Create StorageClass in Kubernetes using EFS. +- Create Weaviate Volumes, where each volume has a different AccessPoint for VolumeHandle(as mentioned above). +- Deploy Weaviate. + +This code is an example of a PV for `weaviate-0` Pod: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: weaviate-0 +spec: + capacity: + storage: 8Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: "efs-sc" + csi: + driver: efs.csi.aws.com + volumeHandle: :: + claimRef: + namespace: + name: weaviate-data-weaviate-0 +``` + +For more, general information on running EFS with Fargate, we recommend reading [this AWS blog](https://aws.amazon.com/blogs/containers/running-stateful-workloads-with-amazon-eks-on-aws-fargate-using-amazon-efs/). + +### Using Azure file CSI with Weaviate +The provisioner `file.csi.azure.com` is **not supported** and will lead to file corruptions. Instead, make sure the storage class defined in values.yaml is from provisioner `disk.csi.azure.com`, for example: + +```yaml +storage: + size: 32Gi + storageClassName: managed +``` + +you can get the list of available storage classes in your cluster with: + +``` +kubectl get storageclasses +``` + +## Troubleshooting + +- If you see `No private IP address found, and explicit IP not provided`, set the pod subnet to be in an valid ip address range of the following: + + ``` + 10.0.0.0/8 + 100.64.0.0/10 + 172.16.0.0/12 + 192.168.0.0/16 + 198.19.0.0/16 + ``` + +### Set `CLUSTER_HOSTNAME` if it may change over time + +In some systems, the cluster hostname may change over time. This is known to create issues with a single-node Weaviate deployment. To avoid this, set the `CLUSTER_HOSTNAME` environment variable in the `values.yaml` file to the cluster hostname. + +```yaml +env: + - CLUSTER_HOSTNAME: "node-1" +``` + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/more-resources/migration/_category_.json b/docs/deploy/other/migration/_category_.json similarity index 100% rename from docs/weaviate/more-resources/migration/_category_.json rename to docs/deploy/other/migration/_category_.json diff --git a/docs/weaviate/more-resources/migration/archive.md b/docs/deploy/other/migration/archive.md similarity index 100% rename from docs/weaviate/more-resources/migration/archive.md rename to docs/deploy/other/migration/archive.md diff --git a/docs/weaviate/more-resources/migration/index.md b/docs/deploy/other/migration/index.md similarity index 100% rename from docs/weaviate/more-resources/migration/index.md rename to docs/deploy/other/migration/index.md diff --git a/docs/weaviate/more-resources/migration/weaviate-1-25.md b/docs/deploy/other/migration/weaviate-1-25.md similarity index 100% rename from docs/weaviate/more-resources/migration/weaviate-1-25.md rename to docs/deploy/other/migration/weaviate-1-25.md diff --git a/docs/weaviate/more-resources/migration/weaviate-1-30.md b/docs/deploy/other/migration/weaviate-1-30.md similarity index 100% rename from docs/weaviate/more-resources/migration/weaviate-1-30.md rename to docs/deploy/other/migration/weaviate-1-30.md diff --git a/docs/deploy/production/Kubernetes/index.mdx b/docs/deploy/production/Kubernetes/index.mdx new file mode 100644 index 00000000..d9d9bca9 --- /dev/null +++ b/docs/deploy/production/Kubernetes/index.mdx @@ -0,0 +1,25 @@ +--- + +sidebar_label: "Kubernetes" + +--- + +# Kubernetes + +Weaviate leverages Kubernetes for scalable, resilient, and production-grade deployments. +This section provides guides, best practices, and tutorials for making the best of your Kubernetes deployments in production. + +## Deployment Options + +Weaviate can use Kubernetes in a variety of deployments for your enterprise environment: + +### Enterprise Cloud + +- Amazon EKS (Elastic Kubernetes Service) +- Google Kubernetes Engine (GKE) +- Azure Kubernetes Service (AKS) + +### Bring Your Own Cloud + +- Amazon EKS (Elastic Kubernetes Service) + diff --git a/docs/deploy/production/Kubernetes/k8s-poc.md b/docs/deploy/production/Kubernetes/k8s-poc.md new file mode 100644 index 00000000..fabb9cfd --- /dev/null +++ b/docs/deploy/production/Kubernetes/k8s-poc.md @@ -0,0 +1,154 @@ +# Building with Weaviate: Getting to Production + +## Introduction + +Are you ready to deploy and test Weaviate on a self-managed K8s (Kubernetes) cluster? This guide shows how to validate Weaviate’s capabilities in your enterprise environment. + +At the end of this guide, expect to have: + +- A configured Helm-based deployment and networking setup +- Basic scaling, persistent storage, and resource management +- TLS, RBAC, and security best practices implements +- Monitoring, logging, and backup strategies enabled + +### Prerequisites + +Before beginning, ensure that you have the following: + +#### Technical Knowledge + +- Basic Kubernetes and containerization conceptual knowledge +- Basic experience with Helm and `kubectl` + +:::note + +Check out the Academy course [“Run Weaviate on Kubernetes”](https://weaviate.io/developers/academy/deployment/k8s) if you need assistance. + +::: + +#### Required Tools + +- A running Kubernetes cluster with Weaviate installed +- `kubectl` installed +- Helm installed + +## Step 1: Configure your Helm Chart + +- Use the official [Weaviate Helm chart](https://github.com/weaviate/weaviate-helm) for your installation: + +``` + helm repo add weaviate https://weaviate.github.io/weaviate-helm + helm install my-weaviate weaviate/weaviate +``` + +- Customize the values to fit your enterprise requirements (e.g., resource allocation, storage settings). +- Deploy the chart and verify pod health. + +## Step 2: Network Security + +- Configure an ingress controller to securely expose Weaviate. +- Enable TLS with a certificate manager and enforce TLS encryption for all client-server communication. +- Assign a domain name for external access. +- Implement RBAC or admin lists to restrict user access. + +
+ An example of RBAC enabled on your Helm chart + +```yaml + authorization: + rbac: + enabled: true + root_users: + - admin_user1 + - admin_user2 +``` +
+ +
+ An example of admin lists implemented on your Helm chart (if not using RBAC) + +```yaml + admin_list: + enabled: true + users: + - admin_user1 + - admin_user2 + - api-key-user-admin + read_only_users: + - readonly_user1 + - readonly_user2 + - api-key-user-readOnly +``` +[Admin List Configuration](/docs/weaviate/configuration/authorization.md#admin-list-kubernetes) + +
+ +:::tip +Using an admin list will allow you to define your admin or read-only user/API-key pairs across all Weaviate resources. Whereas RBAC allows you more granular permissions by defining roles and assigning them to users either via API keys or OIDC. +::: + +## Step 3: Scaling + +- Implement horizontal scaling to ensure high availability: + +```yaml +replicaCount: 3 +``` + +- Define CPU/memory limits and requests to optimize pod efficiency. + +
+ An example of defining CPU and memory limits and cores + +```yaml +resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "4Gi" +``` +
+ +## Step 4: Monitoring and Logging + +- Use Prometheus and Grafana to collect and analyze performance metrics. +- Implement alerting for issue resolution. + +
+ An example of enabling service monitoring + +```yaml +serviceMonitor: + enabled: true + interval: 30s + scrapeTimeout: 10s +``` +
+ + +## Step 5: Upgrades and Backups + +- Use the rolling update strategy used by Helm to minimize downtime. + +
+ An example of configuring the rolling update strategy. + +```yaml +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 +``` +
+ +- Test new Weaviate versions before deploying into production. +- Implement disaster recovery procedures to ensure that data is restored quickly. + +### Conclusion + +Voila! You now have a deployment that is *somewhat* ready for production. Your next step will be to complete the self-assessment and identify any gaps. + +### Next Steps: [Production Readiness Self-Assessment](./production-readiness.md) \ No newline at end of file diff --git a/docs/deploy/production/Kubernetes/production-readiness.md b/docs/deploy/production/Kubernetes/production-readiness.md new file mode 100644 index 00000000..751338e1 --- /dev/null +++ b/docs/deploy/production/Kubernetes/production-readiness.md @@ -0,0 +1,88 @@ +--- +sidebar_label: Production Readiness Self-Assessment +--- + +# Kubernetes Production Readiness Self-Assessment + +Think you’re ready for production? Ensuring that your Weaviate cluster is production-ready requires careful planning, configuration, and ongoing maintenance. Ensuring that you have a stable, reliable deployment requires you to think of your *ending* at the *beginning.* This guide provides you with introspective questions to assess readiness and identify any potential gaps before moving your workloads into production. + +:::tip +If you *do* identify gaps within your deployment, be sure to reach out to your SE (solutions engineer) who can help steer you on the path to production success! +::: + + +### High Availability and Resilience + +- [ ] Are your clusters deployed across multiple availability zones (AZs) or regions to prevent downtime? +- [ ] Are you running Weaviate in a highly available setup with a 3-node minimum? +- [ ] Have you configured your schema to use a replication factor of 3 to ensure copies of data are available during node outage? +- [ ] Are replicas deployed across multiple nodes for redundancy? +- [ ] Is your control plane highly available? +- [ ] Is your application fault-tolerant *without* your control plane? +- [ ] Are there automatic node repair or self-healing mechanisms in place? +- [ ] Have failover scenarios been tested to validate resilience? +- [ ] Are you utilizing Weaviate’s backup capabilities for disaster recovery? + - [ ] How often are these mechanisms tested? + - [ ] Has the ability to recover from a node failure or database corruption been tested? +- [ ] Have you thought about the retention period of backups? + - [ ] How do you clean up any out-of-date backups? +- [ ] Are rolling updates performed to avoid downtime? +- [ ] Are canary deployments implemented to safely test new releases? +- [ ] Do you have development or test environments to safely test changes? + +### Data Ingestion and Query Performance + +- [ ] Is there a strategy for handling heavy ingestion loads? +- [ ] Has the percentage of resources for indexing vs querying applications been specified? +- [ ] Is there a defined strategy for data deduplication and cleanup before ingestion? +- [ ] How frequently is data added, updated, or deleted? + - [ ] Is data updated in place or mostly append-only + - [ ] How often do deletion operations trigger garbage collection? +- [ ] Have you implemented a scheduling strategy for large ingestion jobs? +- [ ] Have you tested query performance under load? + - [ ] Is query performance monitored using Prometheus or Grafana? +- [ ] Have replica shards been deployed for load balancing and failover support? + + +### Resource Management + +- [ ] Have you considered your data’s consumption pattern(s)? + - [ ] Has your memory allocation been right-sized to match workload demand? + - [ ] Has your storage/compute allocation also been right-sized to match workload demand? + - [ ] Is there a process to delete old or unused objects? +- [ ] Have multiple replicas been configured to balance read-heavy workloads? +- [ ] Has the proper storage class been selected for your needs? + - [ ] Does your storage class support volume expansion so that you can support growth over time? +- [ ] Is the data within your cluster properly backed up, including the persistent storage? +- [ ] Is the sharding strategy aligned with the size and access patterns of the dataset? +- [ ] Is `GOMEMLIMIT` properly configured for memory management? + - [ ] Is `GOMEMLIMIT` set based on available system memory to prevent excessive garbage collection pauses? +- [ ] Have you considered vector quantization techniques to reduce memory requirements? + +### Tenant State Management + +- [ ] Are you implementing multi-tenancy? + - [ ] Are there limits or quotas per tenant to avoid noisy neighbor issues? +- [ ] Is there a strategy for offloading inactive tenant data? + +### Security + +- [ ] Are the components of your cluster communicating via SSL/TLS and trusted certificates? +- [ ] Is the *“principle of least privilege”* being followed? +- [ ] Are your container security defaults set properly? +- [ ] Is access to your cluster strictly limited? +- [ ] Has [RBAC](/docs/weaviate/configuration/rbac/index.mdx) been implemented to restrict access? +- [ ] Have network policies been implemented to limit pod-to-pod communication? +- [ ] Are secrets secured with K8s Secrets or a vault solution? +- [ ] Do you have a process for when secrets are exposed, when access is lost to a key or certificate, and when secrets need to be rotated? + +### Monitoring and Observability + +- [ ] Is logging implemented? + - [ ] Are the collected logs stored centrally? +- [ ] Is metric collection enabled using Prometheus (or Alloy, DataDog, or another monitoring platform)? +- [ ] Are health and performance metrics being visualized in Grafana? +- [ ] Are alerts configured for events? + +Evaluate these key areas to build a highly available, resilient, and efficient deployment that will scale to meet your business needs. By ensuring that these self-assessment questions have been addressed, you can proactively identify potential risks and maximize the reliability of your deployment. + diff --git a/docs/deploy/production/index.mdx b/docs/deploy/production/index.mdx new file mode 100644 index 00000000..4f2c75c0 --- /dev/null +++ b/docs/deploy/production/index.mdx @@ -0,0 +1,38 @@ +--- + +sidebar_label: "Overview" +sidebar_position: 0 + +--- + +# Production Environments + +Deploying Weaviate in a production environment requires careful planning to ensure stability, security, and performance. +Currently, the only supported method of deploying Weaviate in a production environment is through Kubernetes. +Unlike development or testing environments, a production instance must be resilient, scalable, and optimized for real-world workloads. + +## Key Aspects of a Production Environments + +- **Scalability and Performance** + - Production workloads demand high availability and low latency. + - Auto-scaling strategies and resource monitoring must be in place. +- **Data Resilience and Backups** + - Data must be safeguarded against corruption and loss. + - Disaster recover strategies and automated backups must be implemented. +- **Security and Compliance** + - Data at rest and in transit should be encrypted. + - liance best practices must be followed. +- **Monitoring and Observability** + - Logs and metrics must be collected for troubleshooting and optimization. + - Monitoring tools should be used to track performance. + - Alerting mechanisms must be set up to detect any anaomolies or failures. +- **Networking and Access Control** + - Privileges should be limited using RBAC (role-based access control). + - Load balancing should be configured. + +### Weaviate Production Environments +Weaviate has different options for deploying in a production environment using: + +- **Serverless Cloud**: A Serverless Cloud instance is a robust cluster designed for production use. +- **Enterprise Cloud**: Enterprise Cloud is a fully-managed deployment of dedicated resources within a secure, highly available environment. +- **Bring Your Own Cloud (BYOC)**: Using your own cloud environment, you are able to have a fully managed deployed. \ No newline at end of file diff --git a/docs/deploy/tutorials/index.mdx b/docs/deploy/tutorials/index.mdx new file mode 100644 index 00000000..54266bad --- /dev/null +++ b/docs/deploy/tutorials/index.mdx @@ -0,0 +1,31 @@ +--- +title: Tutorials +description: Explore Weaviate tutorials for practical guidance on data management and queries. +sidebar_position: 0 +image: og/docs/tutorials.jpg +hide_table_of_contents: true +# tags: ['how to', 'schema'] +--- + +import BasicPrereqs from "/_includes/prerequisites-quickstart.md"; + + + +Tutorials aim to empower you to use Weaviate based on _your_ particular needs. +Here, you can learn how to: + +import CardsSection from "/src/components/CardsSection"; + +export const advancedFeaturesData = [ + { + title: "Set up Role-Based Access Control (RBAC)", + description: + "Configure roles, permissions, and user assignments for secure access control in Weaviate.", + link: "docs/deploy/tutorials/rbac.mdx", + icon: "fas fa-user-shield", + }, +]; + +
+ +
diff --git a/docs/weaviate/tutorials/rbac.mdx b/docs/deploy/tutorials/rbac.mdx similarity index 99% rename from docs/weaviate/tutorials/rbac.mdx rename to docs/deploy/tutorials/rbac.mdx index 7d828ea6..7868eaa2 100644 --- a/docs/weaviate/tutorials/rbac.mdx +++ b/docs/deploy/tutorials/rbac.mdx @@ -1,7 +1,6 @@ --- title: Set up RBAC in Weaviate description: Learn how to set up RBAC (Role Based Access Control) in Weaviate -sidebar_position: 3 image: og/docs/tutorials.jpg # tags: ['basics'] --- diff --git a/docs/weaviate/configuration/authentication.md b/docs/weaviate/configuration/authentication.md index 7a06bbd9..0777f586 100644 --- a/docs/weaviate/configuration/authentication.md +++ b/docs/weaviate/configuration/authentication.md @@ -70,279 +70,6 @@ The way to configure authentication differs by your deployment method, depending For Weaviate Cloud (WCD) instances, authentication is pre-configured with OIDC and API key access. You can [authenticate against Weaviate](../connections/connect-cloud.mdx) with your WCD credentials using OIDC, or [with API keys](/docs/cloud/platform/manage-api-keys). ::: -## API Key Authentication - -API key authentication is a simple and effective way to authenticate users. Each user is assigned a unique API key, which is used to authenticate the user. - -### API keys: Database users - -When [creating database users programatically](./rbac/manage-users.mdx#create-a-user), each user is assigned a distinct API key at creation time. These API keys can also be [regenerated (rotated)](./rbac/manage-users.mdx#rotate-user-api-key). - -### API keys: Docker - -API key authentication can be configured using environment variables. In Docker Compose, set them in the configuration file (`docker-compose.yml`) such as in the following example: - -```yaml -services: - weaviate: - ... - environment: - ... - # Disable anonymous access. - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false' - - # Enables API key authentication. - AUTHENTICATION_APIKEY_ENABLED: 'true' - - # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below. - AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'user-a-key,user-b-key' - - # List one or more user identities, separated by commas. Each identity corresponds to a specific key above. - AUTHENTICATION_APIKEY_USERS: 'user-a,user-b' -``` - -This configuration: -- Disables anonymous access -- Enables API key authentication -- Defines plaintext API keys in `AUTHENTICATION_APIKEY_ALLOWED_KEYS` -- Associates users with the API keys in `AUTHENTICATION_APIKEY_USERS` - -These users can now be assigned permissions based on the authorization settings. - -import DynamicUserManagement from '/_includes/configuration/dynamic-user-management.mdx'; - - - -:::note - -Note that you can either: -- Set one user for all API keys, or -- Define one user per API key (the number of users must match the number of API keys) - -Make sure all listed users are also configured in the authorization settings. - -::: - -### API keys: Kubernetes - -For Kubernetes deployments using Helm, API key authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration: - -```yaml -authentication: - anonymous_access: - # Disable anonymous access. - enabled: false - - apikey: - # Enables API key authentication. - enabled: true - - # List one or more keys in plaintext separated by commas. Each key corresponds to a specific user identity below. - allowed_keys: - - user-a-key - - user-b-key - - # List one or more user identities, separated by commas. Each identity corresponds to a specific key above. - users: - - user-a - - user-b -``` - -This configuration: -- Disables anonymous access -- Enables API key authentication -- Defines plaintext API keys in `allowed_keys` -- Associates users with the API keys in `users` - -:::warning Environment Variables Take Precedence -If you configure API keys using environment variables, those settings will take precedence over the values in `values.yaml`. To use the Helm values configuration, ensure you have not set the corresponding environment variables. -::: - -For enhanced security in production environments, you can store API keys in Kubernetes secrets and reference them using environment variables instead of storing them as plaintext in the Helm values. - -## OIDC Authentication - -OIDC authentication requires the resource (Weaviate) to validate tokens issued by an identity provider. The identity provider authenticates the user and issues tokens, which are then validated by Weaviate. - -In an example setup, a Weaviate instance acts as the resource, Weaviate Cloud (WCD) acts as the identity provider, and the Weaviate client acts on behalf of the user. - -Any "OpenID Connect" compatible token issuer that implements OpenID Connect Discovery is compatible with Weaviate. - -This document discusses how to configure Weaviate as the resource. - -
- - More about OIDC - - -With [OpenID Connect](https://openid.net/connect/) (based on OAuth2), an -external identity provider and token issuer ('token issuer' hereafter) is responsible for managing users. - -OIDC authentication requires obtaining a valid token from the token issuer so that it can be sent in the header of any request to Weaviate. This applies to both REST and GraphQL requests. - -When Weaviate receives a token (JSON Web Token or JWT), it verifies -that it was indeed signed by the configured token issuer. If the signature is -correct, all contents of the token are trusted, which authenticates the user based on the information in the token. - -
- -:::tip TIP: OIDC and RBAC - -The [user management API](./rbac/manage-users.mdx#oidc-user-permissions-management) allows you to assign custom roles and permissions to OIDC users via [Role-Based Access Control (RBAC)](./rbac/index.mdx). - -::: - -### OIDC: Docker - -To configure Weaviate for OIDC-based authentication, add the following environment variables to your configuration file. - -An example `docker-compose.yml` file looks like this: - -```yaml -services: - weaviate: - ... - environment: - ... - # enabled (optional - defaults to false) turns OIDC auth on. All other fields in - # this section will only be validated if enabled is set to true. - AUTHENTICATION_OIDC_ENABLED: 'true' - - # issuer (required) tells weaviate how to discover the token issuer. This - # endpoint must implement the OpenID Connect Discovery spec, so that weaviate - # can retrieve the issuer's public key. - # - # The example URL below uses the path structure commonly found with keycloak - # where an example realm 'my-weaviate-usecase' was created. The exact - # path structure depends on the token issuer. See the token issuer's documentation - # about which endpoint implements OIDC Discovery. - AUTHENTICATION_OIDC_ISSUER: 'http://my-token-issuer/auth/realms/my-weaviate-usecase' - - # client_id (required unless skip_client_id_check is set to true) tells - # Weaviate to check for a particular OAuth 2.0 client_id in the audience claim. - # This is to prevent that a token which was signed by the correct issuer - # but never intended to be used with Weaviate can be used for authentication. - # - # For more information on what clients are in OAuth 2.0, see - # https://tools.ietf.org/html/rfc6749#section-1.1 - AUTHENTICATION_OIDC_CLIENT_ID: 'my-weaviate-client' - - # username_claim (required) tells Weaviate which claim in the token to use for extracting - # the username. The username will be passed to the authorization plugin. - AUTHENTICATION_OIDC_USERNAME_CLAIM: 'email' - - # skip_client_id_check (optional, defaults to false) skips the client_id - # validation in the audience claim as outlined in the section above. - # Not recommended to set this option as it reduces security, only set this - # if your token issuer is unable to provide a correct audience claim - AUTHENTICATION_OIDC_SKIP_CLIENT_ID_CHECK: 'false' - - # scope (optional) these will be used by clients as default scopes for authentication - AUTHENTICATION_OIDC_SCOPES: '' -``` - -:::info OIDC and Azure -As of November 2022, we were aware of some differences in Microsoft Azure's OIDC implementation compared to others. If you are using Azure and experiencing difficulties, [this external blog post](https://xsreality.medium.com/making-azure-ad-oidc-compliant-5734b70c43ff) may be useful. -::: - -### OIDC: Kubernetes - -For Kubernetes deployments using Helm, OIDC authentication can be configured in the `values.yaml` file under the `authentication` section. Here's an example configuration: - -```yaml -authentication: - anonymous_access: - # Disable anonymous access. - enabled: false - oidc: - # enabled (optional - defaults to false) turns OIDC auth on. All other fields in - # this section will only be validated if enabled is set to true. - enabled: true - - # issuer (required) tells weaviate how to discover the token issuer. This - # endpoint must implement the OpenID Connect Discovery spec, so that weaviate - # can retrieve the issuer's public key. - # - # The example URL below uses the path structure commonly found with keycloak - # where an example realm 'my-weaviate-usecase' was created. The exact - # path structure depends on the token issuer. See the token issuer's documentation - # about which endpoint implements OIDC Discovery. - issuer: 'http://my-token-issuer/auth/realms/my-weaviate-usecase' - - # client_id (required unless skip_client_id_check is set to true) tells - # Weaviate to check for a particular OAuth 2.0 client_id in the audience claim. - # This is to prevent that a token which was signed by the correct issuer - # but never intended to be used with Weaviate can be used for authentication. - # - # For more information on what clients are in OAuth 2.0, see - # https://tools.ietf.org/html/rfc6749#section-1.1 - client_id: 'my-weaviate-client' - - # username_claim (required) tells Weaviate which claim in the token to use for extracting - # the username. The username will be passed to the authorization plugin. - username_claim: 'email' - - # skip_client_id_check (optional, defaults to false) skips the client_id - # validation in the audience claim as outlined in the section above. - # Not recommended to set this option as it reduces security, only set this - # if your token issuer is unable to provide a correct audience claim - skip_client_id_check: 'false' - - # scope (optional) these will be used by clients as default scopes for authentication - scopes: '' - - # groups_claim: '' -``` - -### Note: Configuring the OIDC token issuer - -Configuring the OIDC token issuer is outside the scope of this document, but here are a few options as a starting point: - -- For simple use-cases such as for a single user, you can use Weaviate Cloud (WCD) as the OIDC token issuer. To do so: - - Make sure you have a WCD account (you can [sign up here](https://console.weaviate.cloud/)). - - In the Docker Compose file (e.g. `docker-compose.yml`), specify: - - `https://auth.wcs.api.weaviate.io/auth/realms/SeMI` as the issuer (in `AUTHENTICATION_OIDC_ISSUER`), - - `wcs` as the client id (in `AUTHENTICATION_OIDC_CLIENT_ID`), and - - enable the adminlist (`AUTHORIZATION_ADMINLIST_ENABLED: 'true'`) and add your WCD account email as the user (in `AUTHORIZATION_ADMINLIST_USERS`) . - - `email` as the username claim (in `AUTHENTICATION_OIDC_USERNAME_CLAIM`). - -- If you need a more customizable setup you can use commercial OIDC providers like [Okta](https://www.okta.com/). -- As another alternative, you can run your own OIDC token issuer server, which may be the most complex but also configurable solution. Popular open-source solutions include Java-based [Keycloak](https://www.keycloak.org/) and Golang-based [dex](https://github.com/dexidp/dex). - -:::info -By default, Weaviate validates that the token includes a specified client id in the audience claim. If your token issuer does not support this feature, you can turn it off as outlined in the configuration section below. -::: - -## Anonymous Access - -Weaviate can be configured to accept anonymous requests. This is strongly discouraged except for development or evaluation purposes. - -Users that send requests without explicit authentication are authenticated as `user: anonymous`. - -You can use the authorization plugin to specify which permissions to apply to this `anonymous` user. If anonymous access is disabled altogether, any request without an allowed authentication scheme returns `401 Unauthorized`. - -### Anonymous access: Docker - -To enable anonymous access in Docker Compose, add the following environment variable to your configuration file: - -```yaml -services: - weaviate: - ... - environment: - ... - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' -``` - -### Anonymous access: Kubernetes - -To enable anonymous access in Kubernetes, add the following configuration to your `values.yaml` file: - -```yaml -authentication: - anonymous_access: - enabled: true -``` ## Client-side Usage diff --git a/docs/weaviate/configuration/index.md b/docs/weaviate/configuration/index.md new file mode 100644 index 00000000..942faa08 --- /dev/null +++ b/docs/weaviate/configuration/index.md @@ -0,0 +1,22 @@ +--- +title: How to configure Weaviate +sidebar_position: 0 +image: og/docs/configuration.jpg +# tags: ['configuration'] +--- + +This section shows you how to configure Weaviate. + +- Extend Weaviate's functionality by adding vectorizers and other [modules](./modules.md) +- Configure how Weaviate stores and indexes data + + - [schema](../manage-data/collections.mdx) + - [data types](../config-refs/datatypes.md) + - [distance metrics](../config-refs/distances.md) + +- Manage performance and cost with [vector index properties](/docs/weaviate/config-refs/schema/vector-index) +- [Back up](docs/deploy/config-guides/backups.md) your Weaviate instance +- Control access + - [authentication](./authentication.md) + - [authorization](./authorization.md) +- [Monitor](./monitoring.md) your Weaviate instance diff --git a/docs/weaviate/configuration/persistence.md b/docs/weaviate/configuration/persistence.md index 7ded1689..fe93ea2d 100644 --- a/docs/weaviate/configuration/persistence.md +++ b/docs/weaviate/configuration/persistence.md @@ -72,7 +72,7 @@ services: ### Backups -See [Backups](./backups.md). +See [Backups](docs/deploy/config-guides/backups.md). ## Kubernetes diff --git a/docs/weaviate/installation/kubernetes.md b/docs/weaviate/installation/kubernetes.md index c9b43685..46269b75 100644 --- a/docs/weaviate/installation/kubernetes.md +++ b/docs/weaviate/installation/kubernetes.md @@ -1,5 +1,5 @@ --- -title: Kubernetes +title: Installing Kubernetes sidebar_position: 3 image: og/docs/installation.jpg # tags: ['installation', 'Kubernetes'] @@ -9,7 +9,11 @@ image: og/docs/installation.jpg For a tutorial on how to use [minikube](https://minikube.sigs.k8s.io/docs/) to deploy Weaviate on Kubernetes, see the Weaviate Academy course, [Weaviate on Kubernetes](../../academy/deployment/k8s/index.md). ::: -## Requirements +:::info Deploying to Production +If looking for details and best practices for deploying Weaviate in production, see [Weaviate in Production](docs/deploy/production/Kubernetes/k8s-poc.md) +::: + +## Prerequisites * A recent Kubernetes Cluster (at least version 1.23). If you are in a development environment, consider using the kubernetes cluster that is built into Docker desktop. For more information, see the [Docker documentation](https://docs.docker.com/desktop/kubernetes/). * The cluster needs to be able to provision `PersistentVolumes` using Kubernetes' `PersistentVolumeClaims`. @@ -66,12 +70,11 @@ Local models, such as `text2vec-transformers`, `qna-transformers`, and `img2vec #### Resource limits -Starting in Helm chart version 17.0.1, constraints on module resources are commented out to improve performance. To constrain resources for specific modules, add the constraints in your `values.yaml` file. +In an effort to improve performance, constraints on module resources are commented out. To constrain resources for specific modules, add the constraints in your `values.yaml` file. #### gRPC service configuration -Starting in Helm chart version 17.0.0, the gRPC service is enabled by default. If you use an older Helm chart, edit your `values.yaml` file to enable gRPC. - +gRPC service is enabled by default. If using an older Helm chart, edit your `values.yaml` file to enable gRPC. Check that the `enabled` field is set to `true` and the `type` field to `LoadBalancer`. These settings allow you to access the [gRPC API](https://weaviate.io/blog/grpc-performance-improvements) from outside the Kubernetes cluster. ```yaml @@ -87,7 +90,7 @@ grpcService: #### Authentication and authorization -An example configuration for authentication is shown below. +An example configuration for authentication is shown below: ```yaml authentication: @@ -167,24 +170,19 @@ To upgrade to `1.25` or higher from a pre-`1.25` version, you must delete the de See the [1.25 migration guide for Kubernetes](../more-resources/migration/weaviate-1-25.md) for more details. -## Additional Configuration Help - -- [Cannot list resource "configmaps" in API group when deploying Weaviate k8s setup on GCP](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) -- [Error: UPGRADE FAILED: configmaps is forbidden](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) - ### Using EFS with Weaviate In some circumstances, you may wish, or need, to use EFS (Amazon Elastic File System) with Weaviate. And we note in the case of AWS Fargate, you must create the [PV (persistent volume)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) manually, as the PVC will NOT create a PV for you. To use EFS with Weaviate, you need to: -- Create an EFS file system. -- Create an EFS access point for every Weaviate replica. - - All of the Access Points must have a different root-directory so that Pods do not share the data, otherwise it will fail. -- Create EFS mount targets for each subnet of the VPC where Weaviate is deployed. -- Create StorageClass in Kubernetes using EFS. -- Create Weaviate Volumes, where each volume has a different AccessPoint for VolumeHandle(as mentioned above). -- Deploy Weaviate. +1. Create an EFS file system. +1. Create an EFS access point for every Weaviate replica. + 1. All of the Access Points must have a different root-directory so that Pods do not share the data, otherwise it will fail. +1. Create EFS mount targets for each subnet of the VPC where Weaviate is deployed. +1. Create StorageClass in Kubernetes using EFS. +1. Create Weaviate Volumes, where each volume has a different AccessPoint for VolumeHandle(as mentioned above). +1. Deploy Weaviate. This code is an example of a PV for `weaviate-0` Pod: @@ -212,7 +210,14 @@ spec: For more, general information on running EFS with Fargate, we recommend reading [this AWS blog](https://aws.amazon.com/blogs/containers/running-stateful-workloads-with-amazon-eks-on-aws-fargate-using-amazon-efs/). ### Using Azure file CSI with Weaviate -The provisioner `file.csi.azure.com` is **not supported** and will lead to file corruptions. Instead, make sure the storage class defined in values.yaml is from provisioner `disk.csi.azure.com`, for example: + +:::danger Important + +The provisioner `file.csi.azure.com` is **not supported** and will lead to file corruptions. + +::: + +Instead, make sure the storage class defined in values.yaml is from provisioner `disk.csi.azure.com`, for example: ```yaml storage: @@ -247,6 +252,12 @@ env: - CLUSTER_HOSTNAME: "node-1" ``` +## Additional Configuration Help + +- [Cannot list resource "configmaps" in API group when deploying Weaviate k8s setup on GCP](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) + +- [Error: UPGRADE FAILED: configmaps is forbidden](https://stackoverflow.com/questions/58501558/cannot-list-resource-configmaps-in-api-group-when-deploying-weaviate-k8s-setup) + ## Questions and feedback import DocsFeedback from '/_includes/docs-feedback.mdx'; diff --git a/secondaryNavbar.js b/secondaryNavbar.js index 61c0e53e..0e710ab2 100644 --- a/secondaryNavbar.js +++ b/secondaryNavbar.js @@ -56,7 +56,7 @@ const secondaryNavbarItems = { }, ], }, - /* + deploy: { title: "Deploy", icon: "fa fa-database", @@ -64,10 +64,18 @@ const secondaryNavbarItems = { link: "/docs/deploy", links: [ { label: "Get Started", link: "/docs/deploy", sidebar: "deploySidebar" }, - { label: "AWS", link: "/docs/deploy/aws", sidebar: "deployAwsSidebar" }, - ], + { label: "Configuration Guides", link: "/docs/deploy/config-guides", sidebar: "deployConfigSidebar"}, + /*{ label: "Kubernetes", link: "/docs/deploy/k8s", sidebar: "deployK8sSidebar" },*/ + { label: "Production Guides", link: "/docs/deploy/production", sidebar: "deployProductionSidebar"}, + { label: "Tutorials", link: "/docs/deploy/tutorials", sidebar: "deployTutorialSidebar"}, + /*{ label: "AWS", link: "/docs/deploy/aws", sidebar: "deployAwsSidebar" },*/ + /*{ label: "Scaling Strategies", link: "/docs/deploy/scaling-strategies", sidebar: "deployScalingSidebar"},*/ + /*{ label: "Monitoring and Observability", link: "/docs/deploy/monitoring-obs", sidebar: "deployObservabilitySidebar"},*/ + { label: "FAQs", link: "/docs/deploy/faqs", sidebar: "deployFaqsSidebar"}, + { label: "Other", link: "/docs/deploy/other", sidebar: "deployOtherSidebar"}, + ] }, - */ + agents: { title: "Weaviate Agents", icon: "fa fa-robot", diff --git a/sidebars.js b/sidebars.js index fa4355ef..acdc0e09 100644 --- a/sidebars.js +++ b/sidebars.js @@ -428,9 +428,7 @@ const sidebars = { }, "weaviate/configuration/tenant-offloading", "weaviate/configuration/modules", - "weaviate/configuration/backups", "weaviate/configuration/authentication", - "weaviate/configuration/authorization", { type: "category", label: "RBAC", @@ -439,7 +437,6 @@ const sidebars = { id: "weaviate/configuration/rbac/index", }, items: [ - "weaviate/configuration/rbac/configuration", "weaviate/configuration/rbac/manage-roles", "weaviate/configuration/rbac/manage-users", ], @@ -518,7 +515,6 @@ const sidebars = { id: "weaviate/tutorials/index", }, items: [ - "weaviate/tutorials/rbac", "weaviate/tutorials/multi-vector-embeddings", //"weaviate/tutorials/import", "weaviate/tutorials/cross-references", @@ -564,16 +560,6 @@ const sidebars = { id: "weaviate/config-refs/distances", className: "sidebar-item", }, - { - type: "category", - label: "Environment variables", - className: "sidebar-item", - link: { - type: "doc", - id: "weaviate/config-refs/env-vars/index", - }, - items: ["weaviate/config-refs/env-vars/runtime-config"], - }, { type: "doc", id: "weaviate/config-refs/status", @@ -589,11 +575,6 @@ const sidebars = { id: "weaviate/config-refs/meta", className: "sidebar-item", }, - { - type: "doc", - id: "weaviate/config-refs/nodes", - className: "sidebar-item", - }, { type: "doc", id: "weaviate/config-refs/oidc", @@ -731,7 +712,7 @@ const sidebars = { ], }, ], - othersSidebar: [ + othersSidebar: [ { type: "category", label: "Releases", @@ -740,19 +721,6 @@ const sidebars = { id: "weaviate/release-notes/index", }, items: [ - { - type: "category", - label: "Migration", - link: { - type: "doc", - id: "weaviate/more-resources/migration/index", - }, - items: [ - "weaviate/more-resources/migration/weaviate-1-30", - "weaviate/more-resources/migration/weaviate-1-25", - "weaviate/more-resources/migration/archive", - ], - }, ], }, { @@ -786,12 +754,82 @@ const sidebars = { "weaviate/more-resources/glossary", "weaviate/more-resources/example-datasets", ], - deploySidebar: ["deploy/index"], - deployAwsSidebar: [ + deploySidebar: [ + { + type: "doc", + id: "deploy/index", // document ID + label: "Overview", // sidebar label + }, + { + type: "category", + label: "Installation Guides", + collapsible: true, + collapsed: true, + items: [ + "deploy/installation-guides/k8s-installation", + "deploy/installation-guides/docker-installation", + "deploy/installation-guides/gcp-installation", + "deploy/installation-guides/aws-installation", + "deploy/installation-guides/aws-cli", + ], + }, + ], + deployConfigSidebar: [ + { + type: "autogenerated", + dirName: "deploy/config-guides", + }, + ], + /*deployAwsSidebar: [ { type: "autogenerated", dirName: "deploy/aws", }, + ],*/ + /*deployK8sSidebar: [ + { + type: "autogenerated", + dirName: "deploy/k8s", + }, + ],*/ + deployProductionSidebar: [ + { + type: "autogenerated", + dirName: "deploy/production", + + }, + ], + deployTutorialSidebar: [ + { + type: "autogenerated", + dirName: "deploy/tutorials", + }, + ], + deployFaqsSidebar: [ + { + type: "autogenerated", + dirName: "deploy/faqs", + + }, + ], + deployOtherSidebar: [ + { + type: "autogenerated", + dirName: "deploy/other", + }, + { + type: "category", + label: "Migration", + link: { + type: "doc", + id: "deploy/other/migration/index.md", + }, + items: [ + "deploy/other/migration/weaviate-1-30", + "deploy/other/migration/weaviate-1-25.md", + "deploy/other/migration/archive.md", + ], + }, ], agentsSidebar: [ "agents/index", diff --git a/yarn.lock b/yarn.lock index 6915f50c..5533dba0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1857,6 +1857,21 @@ react-helmet-async "npm:@slorber/react-helmet-async@*" react-loadable "npm:@docusaurus/react-loadable@6.0.0" +"@docusaurus/plugin-client-redirects@^3.7.0": + version "3.7.0" + resolved "https://registry.yarnpkg.com/@docusaurus/plugin-client-redirects/-/plugin-client-redirects-3.7.0.tgz#b5cf92529768c457c01ad350bfc50862c6149463" + integrity sha512-6B4XAtE5ZVKOyhPgpgMkb7LwCkN+Hgd4vOnlbwR8nCdTQhLjz8MHbGlwwvZ/cay2SPNRX5KssqKAlcHVZP2m8g== + dependencies: + "@docusaurus/core" "3.7.0" + "@docusaurus/logger" "3.7.0" + "@docusaurus/utils" "3.7.0" + "@docusaurus/utils-common" "3.7.0" + "@docusaurus/utils-validation" "3.7.0" + eta "^2.2.0" + fs-extra "^11.1.1" + lodash "^4.17.21" + tslib "^2.6.0" + "@docusaurus/plugin-content-blog@3.7.0": version "3.7.0" resolved "https://registry.yarnpkg.com/@docusaurus/plugin-content-blog/-/plugin-content-blog-3.7.0.tgz#7bd69de87a1f3adb652e1473ef5b7ccc9468f47e"