From c6f645d2ba290b8f1aac6bfad8215f150da06932 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 22 Jan 2025 16:08:27 -0800 Subject: [PATCH 01/14] add qe to unified tests --- .../client-side-encryption.md | 2 +- source/unified-test-format/schema-1.23.json | 1133 +++++++++++++++++ .../valid-pass/poc-queryable-encryption.json | 179 +++ .../valid-pass/poc-queryable-encryption.yml | 80 ++ .../unified-test-format.md | 29 +- 5 files changed, 1408 insertions(+), 15 deletions(-) create mode 100644 source/unified-test-format/schema-1.23.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json create mode 100644 source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml diff --git a/source/client-side-encryption/client-side-encryption.md b/source/client-side-encryption/client-side-encryption.md index 2eee97e0c9..1576dea211 100644 --- a/source/client-side-encryption/client-side-encryption.md +++ b/source/client-side-encryption/client-side-encryption.md @@ -159,7 +159,7 @@ supports indexed encrypted fields, which are further processed server-side. Is an umbrella term describing the both CSFLE and Queryable Encryption. -**encryptedFields** + **encryptedFields** A BSON document describing the Queryable Encryption encrypted fields. This is analogous to the JSON Schema in FLE. The following is an example encryptedFields in extended canonical JSON: diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json new file mode 100644 index 0000000000..dc716c7fa5 --- /dev/null +++ b/source/unified-test-format/schema-1.23.json @@ -0,0 +1,1133 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Unified Test Format", + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "schemaVersion", + "tests" + ], + "properties": { + "description": { + "type": "string" + }, + "schemaVersion": { + "$ref": "#/definitions/version" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "createEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/entity" + } + }, + "initialData": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + }, + "tests": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/test" + } + }, + "_yamlAnchors": { + "type": "object", + "additionalProperties": true + } + }, + "definitions": { + "version": { + "type": "string", + "pattern": "^[0-9]+(\\.[0-9]+){1,2}$" + }, + "runOnRequirement": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "maxServerVersion": { + "$ref": "#/definitions/version" + }, + "minServerVersion": { + "$ref": "#/definitions/version" + }, + "topologies": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "single", + "replicaset", + "sharded", + "sharded-replicaset", + "load-balanced" + ] + } + }, + "serverless": { + "type": "string", + "enum": [ + "require", + "forbid", + "allow" + ] + }, + "serverParameters": { + "type": "object", + "minProperties": 1 + }, + "auth": { + "type": "boolean" + }, + "authMechanism": { + "type": "string" + }, + "csfle": { + "type": "boolean" + } + } + }, + "entity": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "client": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "uriOptions": { + "type": "object" + }, + "useMultipleMongoses": { + "type": "boolean" + }, + "observeEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "commandStartedEvent", + "commandSucceededEvent", + "commandFailedEvent", + "poolCreatedEvent", + "poolReadyEvent", + "poolClearedEvent", + "poolClosedEvent", + "connectionCreatedEvent", + "connectionReadyEvent", + "connectionClosedEvent", + "connectionCheckOutStartedEvent", + "connectionCheckOutFailedEvent", + "connectionCheckedOutEvent", + "connectionCheckedInEvent", + "serverDescriptionChangedEvent", + "topologyDescriptionChangedEvent" + ] + } + }, + "ignoreCommandMonitoringEvents": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "storeEventsAsEntities": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/storeEventsAsEntity" + } + }, + "observeLogMessages": { + "type": "object", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "command": { + "$ref": "#/definitions/logSeverityLevel" + }, + "topology": { + "$ref": "#/definitions/logSeverityLevel" + }, + "serverSelection": { + "$ref": "#/definitions/logSeverityLevel" + }, + "connection": { + "$ref": "#/definitions/logSeverityLevel" + } + } + }, + "serverApi": { + "$ref": "#/definitions/serverApi" + }, + "observeSensitiveCommands": { + "type": "boolean" + }, + "autoEncryptOpts": { + "keyVaultNamespace": { + "type": "string" + }, + "bypassAutoEncryption": { + "type": "boolean" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } + } + } + }, + "clientEncryption": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "clientEncryptionOpts" + ], + "properties": { + "id": { + "type": "string" + }, + "clientEncryptionOpts": { + "$ref": "#/definitions/clientEncryptionOpts" + } + } + }, + "database": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client", + "databaseName" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "databaseOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "collection": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database", + "collectionName" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "collectionName": { + "type": "string" + }, + "collectionOptions": { + "$ref": "#/definitions/collectionOrDatabaseOptions" + } + } + }, + "session": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "client" + ], + "properties": { + "id": { + "type": "string" + }, + "client": { + "type": "string" + }, + "sessionOptions": { + "type": "object" + } + } + }, + "bucket": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "database" + ], + "properties": { + "id": { + "type": "string" + }, + "database": { + "type": "string" + }, + "bucketOptions": { + "type": "object" + } + } + }, + "thread": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + } + } + } + } + }, + "logComponent": { + "type": "string", + "enum": [ + "command", + "topology", + "serverSelection", + "connection" + ] + }, + "logSeverityLevel": { + "type": "string", + "enum": [ + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", + "trace" + ] + }, + "clientEncryptionOpts": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyVaultClient", + "keyVaultNamespace", + "kmsProviders" + ], + "properties": { + "keyVaultClient": { + "type": "string" + }, + "keyVaultNamespace": { + "type": "string" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + }, + "keyExpirationMS": { + "type": "integer" + } + } + }, + "kmsProviders": { + "$defs": { + "stringOrPlaceholder": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "$$placeholder" + ], + "properties": { + "$$placeholder": {} + } + } + ] + } + }, + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^aws(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "accessKeyId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "secretAccessKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "sessionToken": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^azure(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "tenantId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientId": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "clientSecret": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "identityPlatformEndpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^gcp(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "email": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "privateKey": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + }, + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^kmip(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "endpoint": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + }, + "^local(:[a-zA-Z0-9_]+)?$": { + "type": "object", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/definitions/kmsProviders/$defs/stringOrPlaceholder" + } + } + } + } + }, + "storeEventsAsEntity": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "events" + ], + "properties": { + "id": { + "type": "string" + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "enum": [ + "PoolCreatedEvent", + "PoolReadyEvent", + "PoolClearedEvent", + "PoolClosedEvent", + "ConnectionCreatedEvent", + "ConnectionReadyEvent", + "ConnectionClosedEvent", + "ConnectionCheckOutStartedEvent", + "ConnectionCheckOutFailedEvent", + "ConnectionCheckedOutEvent", + "ConnectionCheckedInEvent", + "CommandStartedEvent", + "CommandSucceededEvent", + "CommandFailedEvent", + "ServerDescriptionChangedEvent", + "TopologyDescriptionChangedEvent" + ] + } + } + } + }, + "collectionData": { + "type": "object", + "additionalProperties": false, + "required": [ + "collectionName", + "databaseName", + "documents" + ], + "properties": { + "collectionName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "createOptions": { + "type": "object", + "properties": { + "writeConcern": false + } + }, + "documents": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "expectedEventsForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "events" + ], + "properties": { + "client": { + "type": "string" + }, + "eventType": { + "type": "string", + "enum": [ + "command", + "cmap", + "sdam" + ] + }, + "events": { + "type": "array" + }, + "ignoreExtraEvents": { + "type": "boolean" + } + }, + "oneOf": [ + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "command" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "cmap" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCmapEvent" + } + } + } + }, + { + "required": [ + "eventType" + ], + "properties": { + "eventType": { + "const": "sdam" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedSdamEvent" + } + } + } + }, + { + "additionalProperties": false, + "properties": { + "client": { + "type": "string" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedCommandEvent" + } + }, + "ignoreExtraEvents": { + "type": "boolean" + } + } + } + ] + }, + "expectedCommandEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "commandStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reply": { + "type": "object" + }, + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + }, + "commandFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "commandName": { + "type": "string" + }, + "databaseName": { + "type": "string" + }, + "hasServiceId": { + "type": "boolean" + }, + "hasServerConnectionId": { + "type": "boolean" + } + } + } + } + }, + "expectedCmapEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "poolCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "poolClearedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "hasServiceId": { + "type": "boolean" + }, + "interruptInUseConnections": { + "type": "boolean" + } + } + }, + "poolClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCreatedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionReadyEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckOutStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckOutFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + } + } + }, + "connectionCheckedOutEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "connectionCheckedInEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "expectedSdamEvent": { + "type": "object", + "additionalProperties": false, + "maxProperties": 1, + "minProperties": 1, + "properties": { + "serverDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/serverDescription" + }, + "newDescription": { + "$ref": "#/definitions/serverDescription" + } + } + }, + "topologyDescriptionChangedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "previousDescription": { + "$ref": "#/definitions/topologyDescription" + }, + "newDescription": { + "$ref": "#/definitions/topologyDescription" + } + } + }, + "serverHeartbeatStartedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatSucceededEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "serverHeartbeatFailedEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "awaited": { + "type": "boolean" + } + } + }, + "topologyOpeningEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + }, + "topologyClosedEvent": { + "type": "object", + "additionalProperties": false, + "properties": {} + } + } + }, + "serverDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Standalone", + "Mongos", + "PossiblePrimary", + "RSPrimary", + "RSSecondary", + "RSOther", + "RSArbiter", + "RSGhost", + "LoadBalancer", + "Unknown" + ] + } + } + }, + "topologyDescription": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "enum": [ + "Single", + "Unknown", + "ReplicaSetNoPrimary", + "ReplicaSetWithPrimary", + "Sharded", + "LoadBalanced" + ] + } + } + }, + "expectedLogMessagesForClient": { + "type": "object", + "additionalProperties": false, + "required": [ + "client", + "messages" + ], + "properties": { + "client": { + "type": "string" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + }, + "ignoreExtraMessages": { + "type": "boolean" + }, + "ignoreMessages": { + "type": "array", + "items": { + "$ref": "#/definitions/expectedLogMessage" + } + } + } + }, + "expectedLogMessage": { + "type": "object", + "additionalProperties": false, + "required": [ + "level", + "component", + "data" + ], + "properties": { + "level": { + "$ref": "#/definitions/logSeverityLevel" + }, + "component": { + "$ref": "#/definitions/logComponent" + }, + "data": { + "type": "object" + }, + "failureIsRedacted": { + "type": "boolean" + } + } + }, + "collectionOrDatabaseOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "readConcern": { + "type": "object" + }, + "readPreference": { + "type": "object" + }, + "writeConcern": { + "type": "object" + }, + "timeoutMS": { + "type": "integer" + }, + "encryptedFields": { + "type": "object" + } + } + }, + "serverApi": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "string" + }, + "strict": { + "type": "boolean" + }, + "deprecationErrors": { + "type": "boolean" + } + } + }, + "operation": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "object": { + "type": "string" + }, + "arguments": { + "type": "object" + }, + "ignoreResultAndError": { + "type": "boolean" + }, + "expectError": { + "$ref": "#/definitions/expectedError" + }, + "expectResult": {}, + "saveResultAsEntity": { + "type": "string" + } + }, + "allOf": [ + { + "not": { + "required": [ + "expectError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "expectError", + "saveResultAsEntity" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectResult" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "expectError" + ] + } + }, + { + "not": { + "required": [ + "ignoreResultAndError", + "saveResultAsEntity" + ] + } + } + ] + }, + "expectedError": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "isError": { + "type": "boolean", + "const": true + }, + "isClientError": { + "type": "boolean" + }, + "isTimeoutError": { + "type": "boolean" + }, + "errorContains": { + "type": "string" + }, + "errorCode": { + "type": "integer" + }, + "errorCodeName": { + "type": "string" + }, + "errorLabelsContain": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "errorLabelsOmit": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "writeErrors": { + "type": "object" + }, + "writeConcernErrors": { + "type": "array", + "items": { + "type": "object" + } + }, + "errorResponse": { + "type": "object" + }, + "expectResult": {} + } + }, + "test": { + "type": "object", + "additionalProperties": false, + "required": [ + "description", + "operations" + ], + "properties": { + "description": { + "type": "string" + }, + "runOnRequirements": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/runOnRequirement" + } + }, + "skipReason": { + "type": "string" + }, + "operations": { + "type": "array", + "items": { + "$ref": "#/definitions/operation" + } + }, + "expectEvents": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedEventsForClient" + } + }, + "expectLogMessages": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/expectedLogMessagesForClient" + } + }, + "outcome": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/collectionData" + } + } + } + } + } +} \ No newline at end of file diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json new file mode 100644 index 0000000000..c43d22631f --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -0,0 +1,179 @@ +{ + "description": "client bulkWrite updateOne-sort", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": { + "$binary": { + "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", + "subType": "00" + } + } + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0", + "collectionOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + } + ], + "tests": [ + { + "description": "insert, replace, and find with queryable encryption", + "operations": [ + { + "object": "collection0", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedInt": 11 + } + } + }, + { + "object": "collection0", + "name": "replaceOne", + "arguments": { + "filter": { + "encryptedInt": 11 + }, + "replacement": { + "encryptedInt": 22 + } + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 22 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 22 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + } + ] +} diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml new file mode 100644 index 0000000000..d5604cd1ea --- /dev/null +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -0,0 +1,80 @@ +description: client bulkWrite updateOne-sort + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "8.0" + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + kmsProviders: + local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + collectionOptions: + encryptedFields: + fields: + - keyId: + $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + path: 'encryptedInt' + bsonType: 'int' + queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: + provider: local + +tests: + - description: insert, replace, and find with queryable encryption + operations: + - object: *collection0 + name: insertOne + arguments: + document: + _id: 1 + encryptedInt: 11 + - object: *collection0 + name: replaceOne + arguments: + filter: { encryptedInt: 11 } + replacement: { encryptedInt: 22 } + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 22 } + expectResult: + - _id: 1 + encryptedInt: 22 + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } \ No newline at end of file diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 6e56e63544..921ccfbe24 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -534,6 +534,14 @@ The structure of this object is as follows: client. - `serverApi`: Optional [serverApi](#serverapi) object. + + + - `autoEncryptOpts`: Optional object with the following fields: + - `kmsProviders`: The same as in [`clientEncryption`](#entity_clientEncryption). + - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". + - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. + Defaults to `false`. + - `clientEncryption`: Optional object. Defines a ClientEncryption object. @@ -1307,6 +1315,8 @@ The structure of this object is as follows: - `readConcern`: Optional object. See [commonOptions_readConcern](#commonOptions_readConcern). - `readPreference`: Optional object. See [commonOptions_readPreference](#commonOptions_readPreference). - `writeConcern`: Optional object. See [commonOptions_writeConcern](#commonOptions_writeConcern). +- `encryptedFields`: Optional object. See + [Client Side Encryption: `encryptedFields`](../client-side-encryption/client-side-encryption.md#encryptedFields) ### Common Options @@ -2780,7 +2790,7 @@ Contexts where one might encounter a root-level document include: include: - [aggregate](#aggregate) - - [find](#find)) + - [find](#find) - [listCollections](#listcollections), listDatabases, and listIndexes - [listSearchIndexes](#listsearchindexes) - [runCursorCommand](#runcursorcommand) @@ -3522,19 +3532,6 @@ would need to represent streams as entities and support IO operations to directl entity. This may not be worth the added complexity if the existing operations provide adequate test coverage for GridFS implementations. -### Support Client-side Encryption integration tests - -Supporting client-side encryption spec tests will require the following changes to the test format: - -- `json_schema` will need to be specified when creating a collection, via either the collection entity definition or - [initialData](#initialData). -- `key_vault_data` can be expressed via [initialData](#initialData) -- `autoEncryptOpts` will need to be specified when defining a client entity. Preparation of this field may require - reading AWS credentials from environment variables. - -The process for executing tests should not require significant changes, but test files will need to express a dependency -on mongocryptd. - ### Incorporate referenced entity operations into the schema version The [Schema Version](#schema-version) is not impacted by changes to operations defined in other specs and referenced in @@ -3552,6 +3549,10 @@ other specs *and* collating spec changes developed in parallel or during the sam ## Changelog +- 2025-01-21: **Schema version 1.23.** + + Support queryable encryption. + - 2024-11-12: **Schema version 1.22.** Add `keyExpirationMS` to `clientEncryption` entity. From aa51fc3af532d7c8bba4b97584d449b745f7ba4a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 5 Feb 2025 20:40:24 -0800 Subject: [PATCH 02/14] move encryptedFields to initialData --- source/unified-test-format/schema-1.23.json | 25 ++++---- source/unified-test-format/tests/Makefile | 2 +- .../valid-pass/poc-queryable-encryption.json | 64 +++++++++---------- .../valid-pass/poc-queryable-encryption.yml | 26 ++++---- .../unified-test-format.md | 4 +- 5 files changed, 63 insertions(+), 58 deletions(-) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index dc716c7fa5..6dd34a6cda 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -188,14 +188,20 @@ "type": "boolean" }, "autoEncryptOpts": { - "keyVaultNamespace": { - "type": "string" - }, - "bypassAutoEncryption": { - "type": "boolean" - }, - "kmsProviders": { - "$ref": "#/definitions/kmsProviders" + "type": "object", + "required": [ + "kmsProviders" + ], + "properties": { + "keyVaultNamespace": { + "type": "string" + }, + "bypassAutoEncryption": { + "type": "boolean" + }, + "kmsProviders": { + "$ref": "#/definitions/kmsProviders" + } } } } @@ -931,9 +937,6 @@ }, "timeoutMS": { "type": "integer" - }, - "encryptedFields": { - "type": "object" } } }, diff --git a/source/unified-test-format/tests/Makefile b/source/unified-test-format/tests/Makefile index b7b58cd7a6..1a049e72ce 100644 --- a/source/unified-test-format/tests/Makefile +++ b/source/unified-test-format/tests/Makefile @@ -1,4 +1,4 @@ -SCHEMA=../schema-1.22.json +SCHEMA=../schema-1.23.json .PHONY: all invalid valid-fail valid-pass atlas-data-lake versioned-api load-balancers gridfs transactions transactions-convenient-api crud collection-management read-write-concern retryable-reads retryable-writes sessions command-logging-and-monitoring client-side-operations-timeout HAS_AJV diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index c43d22631f..4fa5a350e0 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -1,5 +1,5 @@ { - "description": "client bulkWrite updateOne-sort", + "description": "poc-queryable-encryption", "schemaVersion": "1.23", "runOnRequirements": [ { @@ -17,12 +17,7 @@ "autoEncryptOpts": { "kmsProviders": { "local": { - "key": { - "$binary": { - "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", - "subType": "00" - } - } + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" } } } @@ -32,36 +27,14 @@ "database": { "id": "database0", "client": "client0", - "databaseName": "crud-tests" + "databaseName": "poc-queryable-encryption" } }, { "collection": { "id": "collection0", "database": "database0", - "collectionName": "coll0", - "collectionOptions": { - "encryptedFields": { - "fields": [ - { - "keyId": { - "$binary": { - "base64": "EjRWeBI0mHYSNBI0VniQEg==", - "subType": "04" - } - }, - "path": "encryptedInt", - "bsonType": "int", - "queries": { - "queryType": "equality", - "contention": { - "$numberLong": "0" - } - } - } - ] - } - } + "collectionName": "coll0" } }, { @@ -113,6 +86,33 @@ } } ] + }, + { + "databaseName": "poc-queryable-encryption", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } } ], "tests": [ @@ -160,7 +160,7 @@ "outcome": [ { "collectionName": "coll0", - "databaseName": "crud-tests", + "databaseName": "poc-queryable-encryption", "documents": [ { "_id": 1, diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index d5604cd1ea..9a81a40ed5 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -1,4 +1,4 @@ -description: client bulkWrite updateOne-sort +description: poc-queryable-encryption schemaVersion: "1.23" @@ -13,23 +13,16 @@ createEntities: - commandSucceededEvent autoEncryptOpts: kmsProviders: - local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - database: id: &database0 database0 client: *client0 - databaseName: &database0Name crud-tests + databaseName: &database0Name poc-queryable-encryption - collection: id: &collection0 collection0 database: *database0 collectionName: &collection0Name coll0 - collectionOptions: - encryptedFields: - fields: - - keyId: - $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} - path: 'encryptedInt' - bsonType: 'int' - queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} - database: id: &database1 database1 client: *client0 @@ -51,6 +44,17 @@ initialData: status: 1 masterKey: provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: + fields: + - keyId: + $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + path: 'encryptedInt' + bsonType: 'int' + queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} tests: - description: insert, replace, and find with queryable encryption diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 921ccfbe24..193e6208b0 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -1315,8 +1315,6 @@ The structure of this object is as follows: - `readConcern`: Optional object. See [commonOptions_readConcern](#commonOptions_readConcern). - `readPreference`: Optional object. See [commonOptions_readPreference](#commonOptions_readPreference). - `writeConcern`: Optional object. See [commonOptions_writeConcern](#commonOptions_writeConcern). -- `encryptedFields`: Optional object. See - [Client Side Encryption: `encryptedFields`](../client-side-encryption/client-side-encryption.md#encryptedFields) ### Common Options @@ -3551,7 +3549,7 @@ other specs *and* collating spec changes developed in parallel or during the sam - 2025-01-21: **Schema version 1.23.** - Support queryable encryption. + Support automatic encryption. Add `autoEncryptOpts` to `client` entity. - 2024-11-12: **Schema version 1.22.** From fe8ed1ca43374b1a8811d601cb757f1e1ef6728d Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 7 Feb 2025 13:11:23 -0800 Subject: [PATCH 03/14] Apply suggestions from code review Co-authored-by: Kevin Albertson --- .../tests/valid-pass/poc-queryable-encryption.yml | 8 ++++---- source/unified-test-format/unified-test-format.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 9a81a40ed5..19ad1f3e87 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -3,7 +3,8 @@ description: poc-queryable-encryption schemaVersion: "1.23" runOnRequirements: - - minServerVersion: "8.0" + - minServerVersion: "7.0" + csfle: true createEntities: - client: @@ -37,7 +38,7 @@ initialData: - databaseName: *database1Name collectionName: *collection1Name documents: - - _id: { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + - _id: &keyid { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } creationDate: { $date: { $numberLong: "1641024000000" } } updateDate: { $date: { $numberLong: "1641024000000" } } @@ -50,8 +51,7 @@ initialData: createOptions: encryptedFields: fields: - - keyId: - $binary: {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'} + - keyId: *keyid path: 'encryptedInt' bsonType: 'int' queries: {'queryType': 'equality', 'contention': {'$numberLong': '0'}} diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index 193e6208b0..c5ba483136 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -538,7 +538,7 @@ The structure of this object is as follows: - `autoEncryptOpts`: Optional object with the following fields: - `kmsProviders`: The same as in [`clientEncryption`](#entity_clientEncryption). - - `keyVaultNamespace`: Optional, a namespace to the key vault collection. Defaults to "keyvault.datakeys". + - `keyVaultNamespace`: The same as in [`clientEncryption`](#entity_clientEncryption). - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. Defaults to `false`. From 44d883cb2c1019b5689ad5e0f9561ef672f3e20a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 17:45:09 -0800 Subject: [PATCH 04/14] poc test feedback --- .../valid-pass/poc-queryable-encryption.json | 27 +++++++------ .../valid-pass/poc-queryable-encryption.yml | 40 +++++++++---------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 4fa5a350e0..1b748f698e 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -3,7 +3,8 @@ "schemaVersion": "1.23", "runOnRequirements": [ { - "minServerVersion": "8.0" + "minServerVersion": "7.0", + "csfle": true } ], "createEntities": [ @@ -25,29 +26,29 @@ }, { "database": { - "id": "database0", + "id": "encryptedDB", "client": "client0", "databaseName": "poc-queryable-encryption" } }, { "collection": { - "id": "collection0", - "database": "database0", - "collectionName": "coll0" + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "encrypted" } }, { "database": { - "id": "database1", + "id": "keyvaultDB", "client": "client0", "databaseName": "keyvault" } }, { "collection": { - "id": "collection1", - "database": "database0", + "id": "datakeysColl", + "database": "keyvaultDB", "collectionName": "datakeys" } } @@ -89,7 +90,7 @@ }, { "databaseName": "poc-queryable-encryption", - "collectionName": "coll0", + "collectionName": "encrypted", "documents": [], "createOptions": { "encryptedFields": { @@ -120,7 +121,7 @@ "description": "insert, replace, and find with queryable encryption", "operations": [ { - "object": "collection0", + "object": "encryptedColl", "name": "insertOne", "arguments": { "document": { @@ -130,7 +131,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "replaceOne", "arguments": { "filter": { @@ -142,7 +143,7 @@ } }, { - "object": "collection0", + "object": "encryptedColl", "name": "find", "arguments": { "filter": { @@ -159,7 +160,7 @@ ], "outcome": [ { - "collectionName": "coll0", + "collectionName": "encrypted", "databaseName": "poc-queryable-encryption", "documents": [ { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 19ad1f3e87..6e70956629 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -4,7 +4,7 @@ schemaVersion: "1.23" runOnRequirements: - minServerVersion: "7.0" - csfle: true + csfle: true createEntities: - client: @@ -17,26 +17,26 @@ createEntities: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk - database: - id: &database0 database0 + id: &encryptedDB encryptedDB client: *client0 - databaseName: &database0Name poc-queryable-encryption + databaseName: &encryptedDBName poc-queryable-encryption - collection: - id: &collection0 collection0 - database: *database0 - collectionName: &collection0Name coll0 + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName encrypted - database: - id: &database1 database1 + id: &keyvault keyvaultDB client: *client0 - databaseName: &database1Name keyvault + databaseName: &keyvaultName keyvault - collection: - id: &collection1 collection1 - database: *database0 - collectionName: &collection1Name datakeys + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysName datakeys initialData: - - databaseName: *database1Name - collectionName: *collection1Name + - databaseName: *keyvaultName + collectionName: *datakeysName documents: - _id: &keyid { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } @@ -45,8 +45,8 @@ initialData: status: 1 masterKey: provider: local - - databaseName: *database0Name - collectionName: *collection0Name + - databaseName: *encryptedDBName + collectionName: *encryptedCollName documents: [] createOptions: encryptedFields: @@ -59,18 +59,18 @@ initialData: tests: - description: insert, replace, and find with queryable encryption operations: - - object: *collection0 + - object: *encryptedColl name: insertOne arguments: document: _id: 1 encryptedInt: 11 - - object: *collection0 + - object: *encryptedColl name: replaceOne arguments: filter: { encryptedInt: 11 } replacement: { encryptedInt: 22 } - - object: *collection0 + - object: *encryptedColl name: find arguments: filter: { encryptedInt: 22 } @@ -78,7 +78,7 @@ tests: - _id: 1 encryptedInt: 22 outcome: - - collectionName: *collection0Name - databaseName: *database0Name + - collectionName: *encryptedCollName + databaseName: *encryptedDBName documents: - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } \ No newline at end of file From ad2fd55605c52e333c101c4637b151e1deef6f09 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 17:50:43 -0800 Subject: [PATCH 05/14] make keyVaultNamespace required --- source/unified-test-format/schema-1.23.json | 1 + .../tests/valid-pass/poc-queryable-encryption.json | 1 + .../tests/valid-pass/poc-queryable-encryption.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index 6dd34a6cda..b0744eda8a 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -190,6 +190,7 @@ "autoEncryptOpts": { "type": "object", "required": [ + "keyVaultNamespace", "kmsProviders" ], "properties": { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 1b748f698e..2f055e30a2 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -16,6 +16,7 @@ "commandSucceededEvent" ], "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 6e70956629..896bee2701 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -13,6 +13,7 @@ createEntities: - commandStartedEvent - commandSucceededEvent autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys kmsProviders: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk From 573aac1cc6131aa37d5acef0be00f67c37fcf233 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Tue, 11 Feb 2025 18:47:45 -0800 Subject: [PATCH 06/14] remaining auto encrypt options --- source/unified-test-format/schema-1.23.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index b0744eda8a..fd092b0dc2 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -202,6 +202,21 @@ }, "kmsProviders": { "$ref": "#/definitions/kmsProviders" + }, + "schemaMap": { + "type": "object" + }, + "extraOptions": { + "type": "object" + }, + "encryptedFieldsMap": { + "type": "object" + }, + "bypassQueryAnalysis": { + "type": "boolean" + }, + "keyExpirationMS": { + "type": "integer" } } } From 112d5bd873343ff2c8570150cc875f6ae9062bcb Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Wed, 12 Feb 2025 18:07:58 -0800 Subject: [PATCH 07/14] simple schema for schemaMap and encryptedFieldsMap --- source/unified-test-format/schema-1.23.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/unified-test-format/schema-1.23.json b/source/unified-test-format/schema-1.23.json index fd092b0dc2..d75bc40ba2 100644 --- a/source/unified-test-format/schema-1.23.json +++ b/source/unified-test-format/schema-1.23.json @@ -204,13 +204,19 @@ "$ref": "#/definitions/kmsProviders" }, "schemaMap": { - "type": "object" + "type": "object", + "additionalProperties": { + "type": "object" + } }, "extraOptions": { "type": "object" }, "encryptedFieldsMap": { - "type": "object" + "type": "object", + "additionalProperties": { + "type": "object" + } }, "bypassQueryAnalysis": { "type": "boolean" From f910f658835bf6f7481513c77e5aac7d2255957a Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 15:59:45 -0800 Subject: [PATCH 08/14] convert localSchema.yml --- .../tests/legacy/localSchema.yml | 65 ----------- .../tests/unified/localSchema.yml | 110 ++++++++++++++++++ .../valid-pass/poc-queryable-encryption.json | 4 - .../valid-pass/poc-queryable-encryption.yml | 3 - 4 files changed, 110 insertions(+), 72 deletions(-) delete mode 100644 source/client-side-encryption/tests/legacy/localSchema.yml create mode 100644 source/client-side-encryption/tests/unified/localSchema.yml diff --git a/source/client-side-encryption/tests/legacy/localSchema.yml b/source/client-side-encryption/tests/legacy/localSchema.yml deleted file mode 100644 index 89b4bd51d0..0000000000 --- a/source/client-side-encryption/tests/legacy/localSchema.yml +++ /dev/null @@ -1,65 +0,0 @@ -runOn: - - minServerVersion: "4.1.10" -database_name: &database_name "default" -collection_name: &collection_name "default" - -data: [] -# configure an empty schema -json_schema: {} -key_vault_data: [{'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']}] - -tests: - - description: "A local schema should override" - clientOptions: - autoEncryptOpts: - schemaMap: - "default.default": {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} - kmsProviders: - aws: {} # Credentials filled in from environment. - operations: - - name: insertOne - arguments: - document: &doc0 { _id: 1, encrypted_string: "string0" } - - name: find - arguments: - filter: { _id: 1 } - result: [*doc0] - expectations: - # Then key is fetched from the key vault. - - command_started_event: - command: - find: datakeys - filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} - $db: keyvault - readConcern: { level: "majority" } - command_name: find - - command_started_event: - command: - insert: *collection_name - documents: - - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } - ordered: true - command_name: insert - - command_started_event: - command: - find: *collection_name - filter: { _id: 1 } - command_name: find - outcome: - collection: - # Outcome is checked using a separate MongoClient without auto encryption. - data: - - *doc0_encrypted - - description: "A local schema with no encryption is an error" - clientOptions: - autoEncryptOpts: - schemaMap: - "default.default": {'properties': {'test': {'bsonType': 'string'}}, 'bsonType': 'object', 'required': ['test']} - kmsProviders: - aws: {} # Credentials filled in from environment. - operations: - - name: insertOne - arguments: - document: { _id: 1, encrypted_string: "string0" } - result: - errorContains: "JSON schema keyword 'required' is only allowed with a remote schema" \ No newline at end of file diff --git a/source/client-side-encryption/tests/unified/localSchema.yml b/source/client-side-encryption/tests/unified/localSchema.yml new file mode 100644 index 0000000000..a8e9c23648 --- /dev/null +++ b/source/client-side-encryption/tests/unified/localSchema.yml @@ -0,0 +1,110 @@ +description: localSchema + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "4.1.10" + - csfle: true + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'encrypted_w_altname': {'encrypt': {'keyId': '/altname', 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}, 'random': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Random'}}, 'encrypted_string_equivalent': {'encrypt': {'keyId': [{'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}], 'bsonType': 'string', 'algorithm': 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic'}}}, 'bsonType': 'object'} + kmsProviders: + aws: {} # Credentials filled in from environment. + observeEvents: [ commandStartedEvent ] + - client: + id: &client1 client1 + autoEncryptOpts: + schemaMap: + "default.default": {'properties': {'test': {'bsonType': 'string'}}, 'bsonType': 'object', 'required': ['test']} + kmsProviders: + aws: {} # Credentials filled in from environment. + observeEvents: [ commandStartedEvent ] + - database: + id: &keyvault keyvaultDB + client: *client0 + databaseName: &keyvaultDBName keyvault + - collection: + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysCollName datakeys + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + # intentionally the same DB and collection name as encryptedDB/Coll + - database: + id: &encryptedDB2 encryptedDB2 + client: *client1 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl2 encryptedColl2 + database: *encryptedDB2 + collectionName: &encryptedCollName default + +initialData: + - databaseName: *keyvaultDBName + collectionName: *datakeysCollName + documents: + - {'status': 1, '_id': {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}}, 'masterKey': {'provider': 'aws', 'key': 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0', 'region': 'us-east-1'}, 'updateDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyMaterial': {'$binary': {'base64': 'AQICAHhQNmWG2CzOm1dq3kWLM+iDUZhEqnhJwH9wZVpuZ94A8gEqnsxXlR51T5EbEVezUqqKAAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDHa4jo6yp0Z18KgbUgIBEIB74sKxWtV8/YHje5lv5THTl0HIbhSwM6EqRlmBiFFatmEWaeMk4tO4xBX65eq670I5TWPSLMzpp8ncGHMmvHqRajNBnmFtbYxN3E3/WjxmdbOOe+OXpnGJPcGsftc7cB2shRfA4lICPnE26+oVNXT6p0Lo20nY5XC7jyCO', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1552949630483'}}, 'keyAltNames': ['altname', 'another_altname']} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + validator: {} + +tests: + - description: "A local schema should override" + operations: + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + - object: *encryptedColl + name: find + arguments: + filter: { _id: 1 } + expectResult: [*doc0] + expectEvents: + # Then key is fetched from the key vault. + - client: *client0 + events: + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: {"$or": [{"_id": {"$in": [ {'$binary': {'base64': 'AAAAAAAAAAAAAAAAAAAAAA==', 'subType': '04'}} ] }}, {"keyAltNames": {"$in": []}}]} + readConcern: { level: "majority" } + - commandStartedEvent: + commandName: insert + command: + insert: *encryptedCollName + documents: + - &doc0_encrypted { _id: 1, encrypted_string: {'$binary': {'base64': 'AQAAAAAAAAAAAAAAAAAAAAACwj+3zkv2VM+aTfk60RqhXq6a/77WlLwu/BxXFkL7EppGsju/m8f0x5kBDD3EZTtGALGXlym5jnpZAoSIkswHoA==', 'subType': '06'}} } + ordered: true + - commandStartedEvent: + commandName: find + command: + find: *encryptedCollName + filter: { _id: 1 } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0_encrypted + - description: "A local schema with no encryption is an error" + operations: + - object: *encryptedColl2 + name: insertOne + arguments: + document: &doc0 { _id: 1, encrypted_string: "string0" } + expectError: + isClientError: true \ No newline at end of file diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json index 2f055e30a2..2d660c1729 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.json @@ -11,10 +11,6 @@ { "client": { "id": "client0", - "observeEvents": [ - "commandStartedEvent", - "commandSucceededEvent" - ], "autoEncryptOpts": { "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { diff --git a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml index 896bee2701..1ec34beea7 100644 --- a/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml +++ b/source/unified-test-format/tests/valid-pass/poc-queryable-encryption.yml @@ -9,9 +9,6 @@ runOnRequirements: createEntities: - client: id: &client0 client0 - observeEvents: - - commandStartedEvent - - commandSucceededEvent autoEncryptOpts: keyVaultNamespace: keyvault.datakeys kmsProviders: From 364eb4c8a69f243c7b622fdb2512024c49a54cfb Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 16:23:42 -0800 Subject: [PATCH 09/14] convert fle2v2-EncryptedFields-vs-EncryptedFieldsMap --- ...-EncryptedFields-vs-EncryptedFieldsMap.yml | 79 ----- ...EncryptedFields-vs-EncryptedFieldsMap.json | 272 ++++++++++++++++++ ...-EncryptedFields-vs-EncryptedFieldsMap.yml | 119 ++++++++ 3 files changed, 391 insertions(+), 79 deletions(-) delete mode 100644 source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml create mode 100644 source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json create mode 100644 source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml diff --git a/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml b/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml deleted file mode 100644 index 8767132e62..0000000000 --- a/source/client-side-encryption/tests/legacy/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Requires libmongocrypt 1.8.0. -runOn: - - minServerVersion: "7.0.0" - # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol. - # FLE 2 Encrypted collections are not supported on standalone. - topology: [ "replicaset", "sharded", "load-balanced" ] -database_name: &database_name "default" -collection_name: &collection_name "default" -data: [] -encrypted_fields: {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} -key_vault_data: [ {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}}] -tests: - - description: "encryptedFieldsMap is preferred over remote encryptedFields" - clientOptions: - autoEncryptOpts: - kmsProviders: - local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} - encryptedFieldsMap: { - "default.default": { - "fields": [] - } - } - operations: - # EncryptedFieldsMap overrides remote encryptedFields. - # Automatic encryption does not occur on encryptedUnindexed. The value is validated on the server. - - name: insertOne - arguments: - document: &doc0 { - _id: 1, - encryptedUnindexed: { - "$binary": { - "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", - "subType": "06" - } - } - } - - name: find - arguments: - filter: { "_id": 1 } - result: [{"_id": 1, "encryptedUnindexed": "value123" }] - expectations: - - command_started_event: - command: - insert: *collection_name - documents: - - *doc0 - ordered: true - command_name: insert - - command_started_event: - command: - find: *collection_name - filter: { "_id": 1} - command_name: find - - command_started_event: - command: - find: datakeys - filter: { - "$or": [ - { - "_id": { - "$in": [ - {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}} - ] - } - }, - { - "keyAltNames": { - "$in": [] - } - } - ] - } - $db: keyvault - readConcern: { level: "majority" } - command_name: find - outcome: - collection: - data: - - *doc0 \ No newline at end of file diff --git a/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json new file mode 100644 index 0000000000..6c26847526 --- /dev/null +++ b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.json @@ -0,0 +1,272 @@ +{ + "description": "fle2v2-EncryptedFields-vs-EncryptedFieldsMap", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "7.0.0", + "topologies": [ + "replicaset", + "sharded", + "load-balanced" + ] + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": { + "$binary": { + "base64": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk", + "subType": "00" + } + } + } + }, + "encryptedFieldsMap": { + "default.default": { + "fields": [] + } + } + }, + "observeEvents": [ + "commandStartedEvent" + ] + } + }, + { + "database": { + "id": "keyvaultDB", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "datakeysColl", + "database": "keyvaultDB", + "collectionName": "datakeys" + } + }, + { + "database": { + "id": "encryptedDB", + "client": "client0", + "databaseName": "default" + } + }, + { + "collection": { + "id": "encryptedColl", + "database": "encryptedDB", + "collectionName": "default" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyMaterial": { + "$binary": { + "base64": "HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1648914851981" + } + }, + "status": { + "$numberInt": "0" + }, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "default", + "collectionName": "default", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedIndexed", + "bsonType": "string", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + }, + { + "keyId": { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedUnindexed", + "bsonType": "string" + } + ] + } + } + } + ], + "tests": [ + { + "description": "encryptedFieldsMap is preferred over remote encryptedFields", + "operations": [ + { + "object": "encryptedColl", + "name": "insertOne", + "arguments": { + "document": { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + } + }, + { + "object": "encryptedColl", + "name": "find", + "arguments": { + "filter": { + "_id": 1 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedUnindexed": "value123" + } + ] + } + ], + "expectEvents": [ + { + "client": "client0", + "events": [ + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "insert", + "command": { + "insert": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ], + "ordered": true + } + } + }, + { + "commandStartedEvent": { + "databaseName": "default", + "commandName": "find", + "command": { + "find": "default", + "filter": { + "_id": 1 + } + } + } + }, + { + "commandStartedEvent": { + "databaseName": "keyvault", + "commandName": "find", + "command": { + "find": "datakeys", + "filter": { + "$or": [ + { + "_id": { + "$in": [ + { + "$binary": { + "base64": "q83vqxI0mHYSNBI0VniQEg==", + "subType": "04" + } + } + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + }, + "$db": "keyvault", + "readConcern": { + "level": "majority" + } + } + } + } + ] + } + ], + "outcome": [ + { + "collectionName": "default", + "databaseName": "default", + "documents": [ + { + "_id": 1, + "encryptedUnindexed": { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + ] + } + ] + } + ] +} diff --git a/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml new file mode 100644 index 0000000000..8655afe5a9 --- /dev/null +++ b/source/client-side-encryption/tests/unified/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml @@ -0,0 +1,119 @@ +description: fle2v2-EncryptedFields-vs-EncryptedFieldsMap + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "7.0.0" + # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol. + # FLE 2 Encrypted collections are not supported on standalone. + topologies: [ "replicaset", "sharded", "load-balanced" ] + +createEntities: + - client: + id: &client0 client0 + autoEncryptOpts: + kmsProviders: + local: {'key': {'$binary': {'base64': 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'subType': '00'}}} + encryptedFieldsMap: { + "default.default": { + "fields": [] + } + } + observeEvents: [ commandStartedEvent ] + - database: + id: &keyvault keyvaultDB + client: *client0 + databaseName: &keyvaultDBName keyvault + - collection: + id: &datakeys datakeysColl + database: *keyvault + collectionName: &datakeysCollName datakeys + - database: + id: &encryptedDB encryptedDB + client: *client0 + databaseName: &encryptedDBName default + - collection: + id: &encryptedColl encryptedColl + database: *encryptedDB + collectionName: &encryptedCollName default + # intentionally the same DB and collection name as encryptedDB/Coll + +initialData: + - databaseName: *keyvaultDBName + collectionName: *datakeysCollName + documents: + - {'_id': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'keyMaterial': {'$binary': {'base64': 'HBk9BWihXExNDvTp1lUxOuxuZK2Pe2ZdVdlsxPEBkiO1bS4mG5NNDsQ7zVxJAH8BtdOYp72Ku4Y3nwc0BUpIKsvAKX4eYXtlhv5zUQxWdeNFhg9qK7qb8nqhnnLeT0f25jFSqzWJoT379hfwDeu0bebJHr35QrJ8myZdPMTEDYF08QYQ48ShRBli0S+QzBHHAQiM2iJNr4svg2WR8JSeWQ==', 'subType': '00'}}, 'creationDate': {'$date': {'$numberLong': '1648914851981'}}, 'updateDate': {'$date': {'$numberLong': '1648914851981'}}, 'status': {'$numberInt': '0'}, 'masterKey': {'provider': 'local'}} + - databaseName: *encryptedDBName + collectionName: *encryptedCollName + documents: [] + createOptions: + encryptedFields: {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedIndexed', 'bsonType': 'string', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}, {'keyId': {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedUnindexed', 'bsonType': 'string'}]} + +tests: + - description: "encryptedFieldsMap is preferred over remote encryptedFields" + operations: + # EncryptedFieldsMap overrides remote encryptedFields. + # Automatic encryption does not occur on encryptedUnindexed. The value is validated on the server. + - object: *encryptedColl + name: insertOne + arguments: + document: &doc0 { + _id: 1, + encryptedUnindexed: { + "$binary": { + "base64": "BqvN76sSNJh2EjQSNFZ4kBICTQaVZPWgXp41I7mPV1rLFTtw1tXzjcdSEyxpKKqujlko5TeizkB9hHQ009dVY1+fgIiDcefh+eQrm3CkhQ==", + "subType": "06" + } + } + } + - object: *encryptedColl + name: find + arguments: + filter: { "_id": 1 } + expectResult: + - {"_id": 1, "encryptedUnindexed": "value123" } + expectEvents: + - client: *client0 + events: + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: insert + command: + insert: *encryptedCollName + documents: + - *doc0 + ordered: true + - commandStartedEvent: + databaseName: *encryptedDBName + commandName: find + command: + find: *encryptedCollName + filter: { "_id": 1} + - commandStartedEvent: + databaseName: *keyvaultDBName + commandName: find + command: + find: *datakeysCollName + filter: { + "$or": [ + { + "_id": { + "$in": [ + {'$binary': {'base64': 'q83vqxI0mHYSNBI0VniQEg==', 'subType': '04'}} + ] + } + }, + { + "keyAltNames": { + "$in": [] + } + } + ] + } + $db: *keyvaultDBName + readConcern: { level: "majority" } + outcome: + - collectionName: *encryptedCollName + databaseName: *encryptedDBName + documents: + - *doc0 \ No newline at end of file From 01505fc1e6a64d4bd2e27487d2f54910af1a1e4e Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Fri, 14 Feb 2025 17:04:37 -0800 Subject: [PATCH 10/14] document auto encrypt fields --- source/unified-test-format/unified-test-format.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/unified-test-format/unified-test-format.md b/source/unified-test-format/unified-test-format.md index c5ba483136..adc3612f5d 100644 --- a/source/unified-test-format/unified-test-format.md +++ b/source/unified-test-format/unified-test-format.md @@ -541,6 +541,11 @@ The structure of this object is as follows: - `keyVaultNamespace`: The same as in [`clientEncryption`](#entity_clientEncryption). - `bypassAutoEncryption`: Optional, a boolean to indicate whether or not auto encryption should be bypassed. Defaults to `false`. + - `schemaMap`: Optional object. Maps namespaces to encrypted fields. + - `encryptedFieldsMap`: Optional object. Specifies which fields to automatically encrypt and the types of queries allowed on those fields. + - `extraOptions`: Optional object. Configuration options for the encryption library. + - `bypassQueryAnalysis`: Optional. Disables analysis of outgoing commands. Defaults to `false`. + - `keyExpirationMS`: Optional integer. Sets how long data encryption keys are cached. Defaults to 60,000. From 87d3c41aabb1c65fb390d0489b1f1e81c0d58ef7 Mon Sep 17 00:00:00 2001 From: mdb-ad Date: Mon, 24 Feb 2025 00:16:28 -0800 Subject: [PATCH 11/14] client bulkWrite test --- .../tests/unified/client-bulkWrite-qe.json | 272 ++++++++++++++++++ .../tests/unified/client-bulkWrite-qe.yml | 112 ++++++++ 2 files changed, 384 insertions(+) create mode 100644 source/crud/tests/unified/client-bulkWrite-qe.json create mode 100644 source/crud/tests/unified/client-bulkWrite-qe.yml diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json new file mode 100644 index 0000000000..4d73d01351 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -0,0 +1,272 @@ +{ + "description": "client bulkWrite with queryable encryption", + "schemaVersion": "1.23", + "runOnRequirements": [ + { + "minServerVersion": "8.0", + "serverless": "forbid" + } + ], + "createEntities": [ + { + "client": { + "id": "client0", + "observeEvents": [ + "commandStartedEvent", + "commandSucceededEvent" + ], + "autoEncryptOpts": { + "kmsProviders": { + "local": { + "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" + } + } + } + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll0" + } + }, + { + "database": { + "id": "database1", + "client": "client0", + "databaseName": "keyvault" + } + }, + { + "collection": { + "id": "collection1", + "database": "database0", + "collectionName": "datakeys" + } + } + ], + "initialData": [ + { + "databaseName": "keyvault", + "collectionName": "datakeys", + "documents": [ + { + "_id": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "keyAltNames": [ + "local_key" + ], + "keyMaterial": { + "$binary": { + "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==", + "subType": "00" + } + }, + "creationDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "updateDate": { + "$date": { + "$numberLong": "1641024000000" + } + }, + "status": 1, + "masterKey": { + "provider": "local" + } + } + ] + }, + { + "databaseName": "crud-tests", + "collectionName": "coll0", + "documents": [], + "createOptions": { + "encryptedFields": { + "fields": [ + { + "keyId": { + "$binary": { + "base64": "EjRWeBI0mHYSNBI0VniQEg==", + "subType": "04" + } + }, + "path": "encryptedInt", + "bsonType": "int", + "queries": { + "queryType": "equality", + "contention": { + "$numberLong": "0" + } + } + } + ] + } + } + } + ], + "_yamlAnchors": { + "namespace": "crud-tests.coll0" + }, + "tests": [ + { + "description": "client bulkWrite QE replaceOne", + "operations": [ + { + "object": "collection0", + "name": "insertMany", + "arguments": { + "documents": [ + { + "_id": 1, + "encryptedInt": 11 + }, + { + "_id": 2, + "encryptedInt": 22 + }, + { + "_id": 3, + "encryptedInt": 33 + } + ] + } + }, + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectResult": { + "insertedCount": 0, + "upsertedCount": 0, + "matchedCount": 1, + "modifiedCount": 1, + "deletedCount": 0 + } + }, + { + "object": "collection0", + "name": "find", + "arguments": { + "filter": { + "encryptedInt": 44 + } + }, + "expectResult": [ + { + "_id": 1, + "encryptedInt": 44 + } + ] + } + ], + "outcome": [ + { + "collectionName": "coll0", + "databaseName": "crud-tests", + "documents": [ + { + "_id": 1, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 2, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + }, + { + "_id": 3, + "encryptedInt": { + "$$type": "binData" + }, + "__safeContent__": { + "$$type": "array" + } + } + ] + } + ] + }, + { + "description": "client bulkWrite QE with multiple replace fails", + "operations": [ + { + "object": "client0", + "name": "clientBulkWrite", + "arguments": { + "models": [ + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 11 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + }, + { + "replaceOne": { + "namespace": "crud-tests.coll0", + "filter": { + "encryptedInt": { + "$eq": 22 + } + }, + "replacement": { + "encryptedInt": 44 + } + } + } + ] + }, + "expectError": { + "isError": true + } + } + ] + } + ] +} diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml new file mode 100644 index 0000000000..fce623a586 --- /dev/null +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -0,0 +1,112 @@ +description: client bulkWrite with queryable encryption + +schemaVersion: "1.23" + +runOnRequirements: + - minServerVersion: "8.0" + serverless: forbid # Serverless does not support bulkWrite: CLOUDP-256344. + +createEntities: + - client: + id: &client0 client0 + observeEvents: + - commandStartedEvent + - commandSucceededEvent + autoEncryptOpts: + kmsProviders: + local: + key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk + - database: + id: &database0 database0 + client: *client0 + databaseName: &database0Name crud-tests + - collection: + id: &collection0 collection0 + database: *database0 + collectionName: &collection0Name coll0 + - database: + id: &database1 database1 + client: *client0 + databaseName: &database1Name keyvault + - collection: + id: &collection1 collection1 + database: *database0 + collectionName: &collection1Name datakeys + + +initialData: + - databaseName: *database1Name + collectionName: *collection1Name + documents: + - _id: &local_key_id { $binary: { base64: EjRWeBI0mHYSNBI0VniQEg==, subType: "04" } } + keyAltNames: ["local_key"] + keyMaterial: { $binary: { base64: sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==, subType: "00" } } + creationDate: { $date: { $numberLong: "1641024000000" } } + updateDate: { $date: { $numberLong: "1641024000000" } } + status: 1 + masterKey: &local_masterkey + provider: local + - databaseName: *database0Name + collectionName: *collection0Name + documents: [] + createOptions: + encryptedFields: &encrypted_fields {'fields': [{'keyId': {'$binary': {'base64': 'EjRWeBI0mHYSNBI0VniQEg==', 'subType': '04'}}, 'path': 'encryptedInt', 'bsonType': 'int', 'queries': {'queryType': 'equality', 'contention': {'$numberLong': '0'}}}]} + +_yamlAnchors: + namespace: &namespace "crud-tests.coll0" + +tests: + - description: client bulkWrite QE replaceOne + operations: + - object: *collection0 + name: insertMany + arguments: + documents: + - { _id: 1, encryptedInt: 11 } + - { _id: 2, encryptedInt: 22 } + - { _id: 3, encryptedInt: 33 } + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + expectResult: + insertedCount: 0 + upsertedCount: 0 + matchedCount: 1 + modifiedCount: 1 + deletedCount: 0 + - object: *collection0 + name: find + arguments: + filter: { encryptedInt: 44 } + expectResult: + - _id: 1 + encryptedInt: 44 + outcome: + - collectionName: *collection0Name + databaseName: *database0Name + documents: + - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - description: client bulkWrite QE with multiple replace fails + operations: + - object: *client0 + name: clientBulkWrite + arguments: + models: + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 11 } } + replacement: { encryptedInt: 44 } + - replaceOne: + namespace: *namespace + filter: { encryptedInt: { $eq: 22 } } + replacement: { encryptedInt: 44 } + expectError: + # Expect error from mongocryptd or crypt_shared + isError: true From bd7a0fc92b236fb38db6a9e636d3b1c5cc321ff7 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Fri, 18 Apr 2025 11:00:18 -0700 Subject: [PATCH 12/14] keyvault namespace --- source/crud/tests/unified/client-bulkWrite-qe.json | 1 + source/crud/tests/unified/client-bulkWrite-qe.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index 4d73d01351..bede25da0d 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -16,6 +16,7 @@ "commandSucceededEvent" ], "autoEncryptOpts": { + "keyVaultNamespace": "keyvault.datakeys", "kmsProviders": { "local": { "key": "Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk" diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index fce623a586..d8ebbc3f4b 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -13,6 +13,7 @@ createEntities: - commandStartedEvent - commandSucceededEvent autoEncryptOpts: + keyVaultNamespace: keyvault.datakeys kmsProviders: local: key: Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk From b2893e776dfadab1b28cd6c7c059b06b9de4c2ee Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:07:37 -0700 Subject: [PATCH 13/14] use unencrypted client to check collection contents --- .../tests/unified/client-bulkWrite-qe.json | 35 +++++++++++++++---- .../tests/unified/client-bulkWrite-qe.yml | 25 +++++++++---- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/source/crud/tests/unified/client-bulkWrite-qe.json b/source/crud/tests/unified/client-bulkWrite-qe.json index bede25da0d..dad3f3950a 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.json +++ b/source/crud/tests/unified/client-bulkWrite-qe.json @@ -39,6 +39,14 @@ "collectionName": "coll0" } }, + { + "client": { + "id": "client1", + "observeEvents": [ + "commandStartedEvent" + ] + } + }, { "database": { "id": "database1", @@ -52,6 +60,20 @@ "database": "database0", "collectionName": "datakeys" } + }, + { + "database": { + "id": "database2", + "client": "client1", + "databaseName": "crud-tests" + } + }, + { + "collection": { + "id": "collection2", + "database": "database2", + "collectionName": "coll0" + } } ], "initialData": [ @@ -189,13 +211,14 @@ "encryptedInt": 44 } ] - } - ], - "outcome": [ + }, { - "collectionName": "coll0", - "databaseName": "crud-tests", - "documents": [ + "object": "collection2", + "name": "find", + "arguments": { + "filter": {} + }, + "expectResult": [ { "_id": 1, "encryptedInt": { diff --git a/source/crud/tests/unified/client-bulkWrite-qe.yml b/source/crud/tests/unified/client-bulkWrite-qe.yml index d8ebbc3f4b..3c29884932 100644 --- a/source/crud/tests/unified/client-bulkWrite-qe.yml +++ b/source/crud/tests/unified/client-bulkWrite-qe.yml @@ -25,6 +25,10 @@ createEntities: id: &collection0 collection0 database: *database0 collectionName: &collection0Name coll0 + - client: + id: &client1 client1 + observeEvents: + - commandStartedEvent - database: id: &database1 database1 client: *client0 @@ -33,6 +37,14 @@ createEntities: id: &collection1 collection1 database: *database0 collectionName: &collection1Name datakeys + - database: + id: &database2 database2 + client: *client1 + databaseName: &database0Name crud-tests + - collection: + id: &collection2 collection2 + database: *database2 + collectionName: &collection0Name coll0 initialData: @@ -87,13 +99,14 @@ tests: expectResult: - _id: 1 encryptedInt: 44 - outcome: - - collectionName: *collection0Name - databaseName: *database0Name - documents: + - object: *collection2 + name: find + arguments: + filter: {} + expectResult: - { _id: 1, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 2, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } + - { _id: 3, encryptedInt: { $$type: binData }, __safeContent__: { $$type: array} } - description: client bulkWrite QE with multiple replace fails operations: - object: *client0 From 28b2ced08b832f7553517b525cad8ef40898de81 Mon Sep 17 00:00:00 2001 From: mdb-ad <198671546+mdb-ad@users.noreply.github.com> Date: Wed, 23 Apr 2025 20:19:53 -0700 Subject: [PATCH 14/14] remove bulkWrite QE error --- source/crud/bulk-write.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/crud/bulk-write.md b/source/crud/bulk-write.md index 285a5bb338..cab2c49509 100644 --- a/source/crud/bulk-write.md +++ b/source/crud/bulk-write.md @@ -634,13 +634,6 @@ write concern containing the following message: > Cannot request unacknowledged write concern and ordered writes -## Auto-Encryption - -If `MongoClient.bulkWrite` is called on a `MongoClient` configured with `AutoEncryptionOpts`, drivers MUST return an -error with the message: "bulkWrite does not currently support automatic encryption". - -This is expected to be removed once [DRIVERS-2888](https://jira.mongodb.org/browse/DRIVERS-2888) is implemented. - ## Command Batching Drivers MUST accept an arbitrary number of operations as input to the `MongoClient.bulkWrite` method. Because the server @@ -917,6 +910,8 @@ error in this specific situation does not seem helpful enough to require size ch ## **Changelog** +- 2025-04-23: Removed the requirement to error when QE is enabled. + - 2024-11-05: Updated the requirements regarding the size validation. - 2024-10-07: Error if `w:0` is used with `ordered=true` or `verboseResults=true`.